[llvm] [InstCombine] Fold align assume into load's !align metadata if possible. (PR #123247)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 18 04:55:43 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/123247
>From 9f93eb2ca663dad237fae48c0ce69f6597fe0415 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Sep 2024 10:50:09 +0100
Subject: [PATCH 1/6] [InstCombine] Remove redundant alignment assumptions.
Use known bits to remove redundant alignment assumptions.
Libc++ now adds alignment assumptions for std::vector::begin() and
std::vector::end(), so I expect we will see quite a bit more
assumptions in C++ [1]. Try to clean up some redundant ones to start
with.
[1] https://github.com/llvm/llvm-project/pull/108961
---
.../InstCombine/InstCombineCalls.cpp | 31 ++++++++++++++++---
.../Transforms/InstCombine/assume-align.ll | 5 +--
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 842881156dc67f..ab628e94235913 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3199,12 +3199,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
// TODO: apply range metadata for range check patterns?
}
- // Separate storage assumptions apply to the underlying allocations, not any
- // particular pointer within them. When evaluating the hints for AA purposes
- // we getUnderlyingObject them; by precomputing the answers here we can
- // avoid having to do so repeatedly there.
for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
OperandBundleUse OBU = II->getOperandBundleAt(Idx);
+
+ // Separate storage assumptions apply to the underlying allocations, not
+ // any particular pointer within them. When evaluating the hints for AA
+ // purposes we getUnderlyingObject them; by precomputing the answers here
+ // we can avoid having to do so repeatedly there.
if (OBU.getTagName() == "separate_storage") {
assert(OBU.Inputs.size() == 2);
auto MaybeSimplifyHint = [&](const Use &U) {
@@ -3218,6 +3219,28 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MaybeSimplifyHint(OBU.Inputs[0]);
MaybeSimplifyHint(OBU.Inputs[1]);
}
+
+ // Try to remove redundant alignment assumptions.
+ if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
+ RetainedKnowledge RK = getKnowledgeFromBundle(
+ *cast<AssumeInst>(II), II->bundle_op_info_begin()[Idx]);
+ if (!RK || RK.AttrKind != Attribute::Alignment ||
+ !isPowerOf2_64(RK.ArgValue))
+ continue;
+
+ // Try to get the instruction before the assumption to use as context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (KnownAlign < RK.ArgValue)
+ continue;
+
+ auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
+ return New;
+ }
}
// Convert nonnull assume like:
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index f0e02574330861..962ad4f792a0d1 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -175,7 +175,6 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt
define ptr @redundant_assume_align_1(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_1(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 1) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -189,7 +188,6 @@ define ptr @redundant_assume_align_1(ptr %p) {
define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]]
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -214,8 +212,7 @@ define ptr @assume_align_16_via_align_metadata(ptr %p) {
define ptr @redundant_assume_align_8_via_align_attribute(ptr align 8 %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_attribute(
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P:%.*]], i32 8) ]
-; CHECK-NEXT: call void @foo(ptr [[P]])
+; CHECK-NEXT: call void @foo(ptr [[P:%.*]])
; CHECK-NEXT: ret ptr [[P]]
;
call void @llvm.assume(i1 true) [ "align"(ptr %p, i32 8) ]
>From 882f44f6d30d36fc0083e7fdc7eaeb519ab3eb1a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Sep 2024 10:50:09 +0100
Subject: [PATCH 2/6] [InstCombine] Fold align assume into load's !align
metadata if possible.
If an alignment assumption is valid in the context of a corresponding
load of the pointer the assumption applies to, the assumption can be
replaced !align metadata on the load.
The benefits of folding it into !align are that existing code makes
better use of !align and it allows removing the now-redundant call
instructions.
!fix formatting.
!fixup check alignment is ConstantInt.
!fixup check align type size.
!fixup use getKnowledgeFromBundle
!fixup update test check
!fixup skip arguments early
---
.../InstCombine/InstCombineCalls.cpp | 31 ++++++++++++++-----
.../Transforms/InstCombine/assume-align.ll | 27 +++++++++++++++-
2 files changed, 49 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ab628e94235913..1ca01f75897d5a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3220,7 +3220,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MaybeSimplifyHint(OBU.Inputs[1]);
}
- // Try to remove redundant alignment assumptions.
+ // Try to fold alignment assumption into a load's !align metadata, if the
+ // assumption is valid in the load's context and remove redundant ones.
if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
RetainedKnowledge RK = getKnowledgeFromBundle(
*cast<AssumeInst>(II), II->bundle_op_info_begin()[Idx]);
@@ -3228,16 +3229,30 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
!isPowerOf2_64(RK.ArgValue))
continue;
- // Try to get the instruction before the assumption to use as context.
- Instruction *CtxI = nullptr;
- if (CtxI && II->getParent()->begin() != II->getIterator())
- CtxI = II->getPrevNode();
+ auto *LI = dyn_cast<LoadInst>(OBU.Inputs[0]);
+ if (LI &&
+ isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
- unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
- if (KnownAlign < RK.ArgValue)
+ if (isa<Argument>(LI->getPointerOperand()))
continue;
+ LI->setMetadata(
+ LLVMContext::MD_align,
+ MDNode::get(II->getContext(), ValueAsMetadata::getConstant(
+ Builder.getInt64(RK.ArgValue))));
+ MDNode *MD = MDNode::get(II->getContext(), {});
+ LI->setMetadata(LLVMContext::MD_noundef, MD);
+ } else {
+ // Try to get the instruction before the assumption to use as context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (KnownAlign < RK.ArgValue)
+ continue;
+ }
auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
return New;
}
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index 962ad4f792a0d1..cd7f5841769f25 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -123,7 +123,6 @@ define i8 @assume_align_non_pow2(ptr %p) {
ret i8 %v
}
-; TODO: Can fold alignment assumption into !align metadata on load.
define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
@@ -135,6 +134,17 @@ define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p)
ret ptr %p2
}
+define ptr @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
+; CHECK-LABEL: @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
+; CHECK-NEXT: ret ptr [[P2]]
+;
+ %p2 = load ptr, ptr %p
+ call void @llvm.assume(i1 true) [ "align"(ptr %p2, i32 8) ]
+ ret ptr %p2
+}
+
define ptr @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(ptr %p) {
; CHECK-LABEL: @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
@@ -175,6 +185,7 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt
define ptr @redundant_assume_align_1(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_1(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 1) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -188,6 +199,7 @@ define ptr @redundant_assume_align_1(ptr %p) {
define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -247,6 +259,19 @@ define ptr @redundant_assume_align_8_via_asume(ptr %p) {
}
declare void @foo(ptr)
+
+; !align must have a constant integer alignment.
+define ptr @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(ptr %p, i64 %align) {
+; CHECK-LABEL: @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 [[ALIGN:%.*]]) ]
+; CHECK-NEXT: ret ptr [[P2]]
+;
+ %p2 = load ptr, ptr %p
+ call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 %align) ]
+ ret ptr %p2
+}
+
;.
; CHECK: [[META0]] = !{i64 8}
;.
>From 1605a8174a64973f412ec464dae335bc777a888f Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 22 Dec 2024 11:16:58 +0000
Subject: [PATCH 3/6] [InstCombine] Add option to clean up some assumptions.
---
.../llvm/Transforms/InstCombine/InstCombine.h | 6 ++++++
llvm/lib/Passes/PassBuilder.cpp | 2 ++
llvm/lib/Passes/PassBuilderPipelines.cpp | 18 ++++++++++++------
llvm/lib/Passes/PassRegistry.def | 2 +-
.../InstCombine/InstCombineCalls.cpp | 3 +--
.../InstCombine/InstCombineInternal.h | 8 ++++++--
.../InstCombine/InstructionCombining.cpp | 4 ++--
7 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
index c12d749709cd25..dd9d05cbbcf7c9 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
@@ -31,6 +31,7 @@ struct InstCombineOptions {
// Verify that a fix point has been reached after MaxIterations.
bool VerifyFixpoint = false;
unsigned MaxIterations = InstCombineDefaultMaxIterations;
+ bool CleanupAssumptions = false;
InstCombineOptions() = default;
@@ -43,6 +44,11 @@ struct InstCombineOptions {
MaxIterations = Value;
return *this;
}
+
+ InstCombineOptions &setCleanupAssumptions(bool Value) {
+ CleanupAssumptions = Value;
+ return *this;
+ }
};
class InstCombinePass : public PassInfoMixin<InstCombinePass> {
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f698a3df08ef78..ebf91c2ff1ca90 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -918,6 +918,8 @@ Expected<InstCombineOptions> parseInstCombineOptions(StringRef Params) {
ParamName).str(),
inconvertibleErrorCode());
Result.setMaxIterations((unsigned)MaxIterations.getZExtValue());
+ } else if (ParamName == "cleanup-assumptions") {
+ Result.setCleanupAssumptions(Enable);
} else {
return make_error<StringError>(
formatv("invalid InstCombine pass parameter '{0}' ", ParamName).str(),
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4ec0fb8fc81ea4..9985b946a54833 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1305,7 +1305,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(LoopLoadEliminationPass());
}
// Cleanup after the loop optimization passes.
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
@@ -1317,7 +1318,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// dead (or speculatable) control flows or more combining opportunities.
ExtraPasses.addPass(EarlyCSEPass());
ExtraPasses.addPass(CorrelatedValuePropagationPass());
- ExtraPasses.addPass(InstCombinePass());
+ ExtraPasses.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
LoopPassManager LPM;
LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
/*AllowSpeculation=*/true));
@@ -1328,7 +1330,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
/*UseBlockFrequencyInfo=*/true));
ExtraPasses.addPass(
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
- ExtraPasses.addPass(InstCombinePass());
+ ExtraPasses.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
FPM.addPass(std::move(ExtraPasses));
}
@@ -1351,7 +1354,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
if (IsFullLTO) {
FPM.addPass(SCCPPass());
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
FPM.addPass(BDCEPass());
}
@@ -1366,7 +1370,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(VectorCombinePass());
if (!IsFullLTO) {
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
// Unroll small loops to hide loop backedge latency and saturate any
// parallel execution resources of an out-of-order processor. We also then
// need to clean up redundancies and loop invariant code.
@@ -1392,7 +1397,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
}
FPM.addPass(InferAlignmentPass());
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
// This is needed for two reasons:
// 1. It works around problems that instcombine introduces, such as sinking
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index a93a995655a147..fd985e46c71667 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -539,7 +539,7 @@ FUNCTION_PASS_WITH_PARAMS(
[](InstCombineOptions Opts) { return InstCombinePass(Opts); },
parseInstCombineOptions,
"no-use-loop-info;use-loop-info;no-verify-fixpoint;verify-fixpoint;"
- "max-iterations=N")
+ "max-iterations=N;cleanup-assumptions")
FUNCTION_PASS_WITH_PARAMS(
"loop-unroll", "LoopUnrollPass",
[](LoopUnrollOptions Opts) { return LoopUnrollPass(Opts); },
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 1ca01f75897d5a..700faf3e9b7e00 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3228,12 +3228,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (!RK || RK.AttrKind != Attribute::Alignment ||
!isPowerOf2_64(RK.ArgValue))
continue;
-
auto *LI = dyn_cast<LoadInst>(OBU.Inputs[0]);
if (LI &&
isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- if (isa<Argument>(LI->getPointerOperand()))
+ if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))
continue;
LI->setMetadata(
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 83e1da98deeda0..b365a41a87d7ac 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -59,6 +59,8 @@ class User;
class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
: public InstCombiner,
public InstVisitor<InstCombinerImpl, Instruction *> {
+ bool CleanupAssumptions = false;
+
public:
InstCombinerImpl(InstructionWorklist &Worklist, BuilderTy &Builder,
bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
@@ -66,9 +68,11 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
DominatorTree &DT, OptimizationRemarkEmitter &ORE,
BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI,
ProfileSummaryInfo *PSI, const DataLayout &DL,
- ReversePostOrderTraversal<BasicBlock *> &RPOT)
+ ReversePostOrderTraversal<BasicBlock *> &RPOT,
+ bool CleanupAssumptions)
: InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE,
- BFI, BPI, PSI, DL, RPOT) {}
+ BFI, BPI, PSI, DL, RPOT),
+ CleanupAssumptions(CleanupAssumptions) {}
virtual ~InstCombinerImpl() = default;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index fb21576722461e..9a3ff72281b963 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -5550,7 +5550,7 @@ static bool combineInstructionsOverFunction(
<< F.getName() << "\n");
InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT,
- ORE, BFI, BPI, PSI, DL, RPOT);
+ ORE, BFI, BPI, PSI, DL, RPOT, Opts.CleanupAssumptions);
IC.MaxArraySizeForCombine = MaxArraySize;
bool MadeChangeInThisIteration = IC.prepareWorklist(F);
MadeChangeInThisIteration |= IC.run();
@@ -5599,7 +5599,7 @@ PreservedAnalyses InstCombinePass::run(Function &F,
FunctionAnalysisManager &AM) {
auto &LRT = AM.getResult<LastRunTrackingAnalysis>(F);
// No changes since last InstCombine pass, exit early.
- if (LRT.shouldSkip(&ID))
+ if (LRT.shouldSkip(&ID) && !Options.CleanupAssumptions)
return PreservedAnalyses::all();
auto &AC = AM.getResult<AssumptionAnalysis>(F);
>From dbe71c77a6e5b2d64859d36cad040b919441513a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 14:59:55 +0000
Subject: [PATCH 4/6] [Mem2Reg] Add align assumption
---
.../Utils/PromoteMemoryToRegister.cpp | 24 +++++++++++++++++++
.../AArch64/infer-align-from-assumption.ll | 14 ++++++-----
2 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 656bb1ebd1161e..a89471adb393c4 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -30,6 +30,7 @@
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugProgramInstruction.h"
@@ -453,6 +454,15 @@ static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) {
AC->registerAssumption(cast<AssumeInst>(CI));
}
+
+static void addAssumeAlign(const DataLayout &DL, LoadInst *LI, Value *Val) {
+ auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+ IRBuilder Builder(LI);
+ Builder.CreateAlignmentAssumption(DL, Val, B);
+}
+
+
static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
const DataLayout &DL, AssumptionCache *AC,
const DominatorTree *DT) {
@@ -473,6 +483,20 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
LI->getMetadata(LLVMContext::MD_noundef) &&
!isKnownNonZero(Val, SimplifyQuery(DL, DT, AC, LI)))
addAssumeNonNull(AC, LI);
+
+ if (AC && LI->getMetadata(LLVMContext::MD_align) &&
+ LI->getMetadata(LLVMContext::MD_noundef)) {
+ auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+
+ auto KB = computeKnownBits(Val, 3, SimplifyQuery(DL, DT, AC, LI));
+ unsigned AlignFromKB = 1 << KB.countMinTrailingZeros();
+ if (AlignFromKB < B->getZExtValue()) {
+ addAssumeAlign(DL, LI, Val);
+}
+
+}
+
}
static void removeIntrinsicUsers(AllocaInst *AI) {
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
index 632e3a56aacac7..2440a61ccaf44f 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
@@ -8,8 +8,7 @@ declare void @llvm.assume(i1 noundef)
define i32 @earlycse_entry(ptr %p) {
; CHECK-LABEL: define i32 @earlycse_entry(
; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
-; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_I]], i64 4) ]
+; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0:![0-9]+]], !noundef [[META1:![0-9]+]]
; CHECK-NEXT: [[L_ASSUME_ALIGNED_I_I:%.*]] = load i32, ptr [[L_I]], align 4
; CHECK-NEXT: [[R_I_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I_I]])
; CHECK-NEXT: [[L_2_I:%.*]] = load ptr, ptr [[P]], align 8
@@ -31,8 +30,7 @@ define i32 @earlycse_entry(ptr %p) {
define i32 @earlycse_fn1(ptr %p) {
; CHECK-LABEL: define i32 @earlycse_fn1(
; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 4) ]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]], !noundef [[META1]]
; CHECK-NEXT: [[L_ASSUME_ALIGNED_I:%.*]] = load i32, ptr [[L]], align 4
; CHECK-NEXT: [[R_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I]])
; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8
@@ -83,8 +81,7 @@ define void @sroa_align_entry(ptr %p) {
define ptr @sroa_fn1(ptr %p) {
; CHECK-LABEL: define ptr @sroa_fn1(
; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 8) ]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META2:![0-9]+]], !noundef [[META1]]
; CHECK-NEXT: [[L_FN3_I_I:%.*]] = load i64, ptr [[L]], align 8
; CHECK-NEXT: [[I_I:%.*]] = inttoptr i64 [[L_FN3_I_I]] to ptr
; CHECK-NEXT: ret ptr [[I_I]]
@@ -118,3 +115,8 @@ define i64 @sroa_fn3(ptr %0) {
%l.fn3 = load i64, ptr %0, align 1
ret i64 %l.fn3
}
+;.
+; CHECK: [[META0]] = !{i64 4}
+; CHECK: [[META1]] = !{}
+; CHECK: [[META2]] = !{i64 8}
+;.
>From 4f80cd7ca9bd6c64459cf79a8c819d781a3d7654 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 15:00:55 +0000
Subject: [PATCH 5/6] DIsable
---
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 700faf3e9b7e00..de27ea1844e50d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3232,8 +3232,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (LI &&
isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))
- continue;
+/* if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))*/
+ /*continue;*/
LI->setMetadata(
LLVMContext::MD_align,
>From 6cca7697b9bb7b66361fc5bfa6ca9f8d8c079eb4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 16:36:49 +0000
Subject: [PATCH 6/6] Step
---
.../InstCombine/InstCombineCalls.cpp | 73 ++++++++++++++++---
llvm/lib/Transforms/Scalar/EarlyCSE.cpp | 21 ++++++
.../Utils/PromoteMemoryToRegister.cpp | 7 +-
.../EarlyCSE/materialize-align-assumptions.ll | 38 ++++++++--
4 files changed, 118 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index de27ea1844e50d..4d8e93437b9bd0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3242,15 +3242,70 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MDNode *MD = MDNode::get(II->getContext(), {});
LI->setMetadata(LLVMContext::MD_noundef, MD);
} else {
- // Try to get the instruction before the assumption to use as context.
- Instruction *CtxI = nullptr;
- if (CtxI && II->getParent()->begin() != II->getIterator())
- CtxI = II->getPrevNode();
-
- auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
- unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
- if (KnownAlign < RK.ArgValue)
- continue;
+ auto *C = dyn_cast<Constant>(RK.WasOn);
+ if (C && C->isNullValue()) {
+ } else {
+ Value *UO = getUnderlyingObject(RK.WasOn);
+
+ bool CanUseAlign = false;
+ SetVector<const Instruction *> WorkList;
+ for (const User *U : RK.WasOn->users())
+ if (auto *I = dyn_cast<Instruction>(U))
+ WorkList.insert(I);
+
+ for (unsigned I = 0; I != WorkList.size(); ++I) {
+ auto *Curr = WorkList[I];
+ if (!DT.dominates(II, Curr))
+ continue;
+ if (auto *LI = dyn_cast<LoadInst>(Curr)) {
+ if (LI->getAlign().value() < RK.ArgValue) {
+ CanUseAlign = true;
+ break;
+ }
+ continue;
+ }
+ if (auto *SI = dyn_cast<StoreInst>(Curr)) {
+ auto *PtrOpI = dyn_cast<Instruction>(SI->getPointerOperand());
+ if (PtrOpI && WorkList.contains(PtrOpI) &&
+ SI->getAlign().value() < RK.ArgValue) {
+ CanUseAlign = true;
+ break;
+ }
+ continue;
+ }
+ if (isa<ReturnInst, CallBase>(Curr)) {
+ CanUseAlign = true;
+ break;
+ }
+ if (isa<ICmpInst>(Curr) &&
+ !isa<Constant>(cast<Instruction>(Curr)->getOperand(0)) &&
+ !isa<Constant>(cast<Instruction>(Curr)->getOperand(1))) {
+ CanUseAlign = true;
+ break;
+ }
+ if (!Curr->getType()->isPointerTy())
+ continue;
+
+ if (WorkList.size() > 16) {
+ CanUseAlign = true;
+ break;
+ }
+ for (const User *U : Curr->users())
+ WorkList.insert(cast<Instruction>(U));
+ }
+ if (CanUseAlign && (!UO || isa<Argument>(UO)))
+ continue;
+ // Try to get the instruction before the assumption to use as
+ // context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (CanUseAlign && KnownAlign < RK.ArgValue)
+ continue;
+ }
}
auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
return New;
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 3a0ae6b01a1144..7ac1df8b3d0560 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -16,6 +16,7 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopedHashTable.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
@@ -31,6 +32,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
@@ -1599,6 +1601,25 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
if (InVal.IsLoad)
if (auto *I = dyn_cast<Instruction>(Op))
combineMetadataForCSE(I, &Inst, false);
+
+ // If the load has align and noundef metadata, preserve it via an
+ // alignment assumption. Note that this doesn't use salavageKnowledge,
+ // as we need to create the assumption for the value we replaced the
+ // load with.
+ if (auto *AlignMD = Inst.getMetadata(LLVMContext::MD_align)) {
+ if (Inst.hasMetadata(LLVMContext::MD_noundef) ||
+ programUndefinedIfPoison(&Inst)) {
+ Inst.setMetadata(LLVMContext::MD_align, nullptr);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+ auto KB = computeKnownBits(Op, 3, SQ.getWithInstruction(&Inst));
+ unsigned AlignFromKB = 1 << KB.countMinTrailingZeros();
+ if (AlignFromKB < B->getZExtValue()) {
+ IRBuilder Builder(&Inst);
+ Builder.CreateAlignmentAssumption(SQ.DL, Op, B);
+ }
+ }
+ }
+
if (!Inst.use_empty())
Inst.replaceAllUsesWith(Op);
salvageKnowledge(&Inst, &AC);
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index a89471adb393c4..93d9767522bb51 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -485,7 +485,8 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
addAssumeNonNull(AC, LI);
if (AC && LI->getMetadata(LLVMContext::MD_align) &&
- LI->getMetadata(LLVMContext::MD_noundef)) {
+ (LI->getMetadata(LLVMContext::MD_noundef) ||
+ programUndefinedIfPoison(LI))) {
auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
@@ -494,9 +495,7 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
if (AlignFromKB < B->getZExtValue()) {
addAssumeAlign(DL, LI, Val);
}
-
-}
-
+ }
}
static void removeIntrinsicUsers(AllocaInst *AI) {
diff --git a/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll b/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
index ea63376957162b..837a73a00d6431 100644
--- a/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
+++ b/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
@@ -3,6 +3,24 @@
declare void @foo(ptr)
+define ptr @align_replacement_does_not_have_align_metadata_missing_noundef(ptr noalias %p) {
+; CHECK-LABEL: define ptr @align_replacement_does_not_have_align_metadata_missing_noundef(
+; CHECK-SAME: ptr noalias [[P:%.*]]) {
+; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: call void @foo(ptr [[L_1]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
+; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: ret ptr [[GEP]]
+;
+ %l.1 = load ptr, ptr %p, align 8
+ call void @foo(ptr %l.1)
+ %l.2 = load ptr, ptr %p, align 8
+ %gep = getelementptr i8, ptr %l.2, i64 4
+ store ptr %gep, ptr %p, align 8
+ %l.3 = load ptr, ptr %p, align 8, !align !0
+ ret ptr %l.3
+}
+
define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
; CHECK-LABEL: define ptr @align_replacement_does_not_have_align_metadata(
; CHECK-SAME: ptr noalias [[P:%.*]]) {
@@ -10,6 +28,7 @@ define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
; CHECK-NEXT: call void @foo(ptr [[L_1]])
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[GEP]], i64 4) ]
; CHECK-NEXT: ret ptr [[GEP]]
;
%l.1 = load ptr, ptr %p, align 8
@@ -17,7 +36,7 @@ define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
%l.2 = load ptr, ptr %p, align 8
%gep = getelementptr i8, ptr %l.2, i64 4
store ptr %gep, ptr %p, align 8
- %l.3 = load ptr, ptr %p, align 8, !align !0
+ %l.3 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.3
}
@@ -27,12 +46,13 @@ define ptr @align_replacement_does_not_have_align_metadata2(ptr noalias %p) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[GEP]], i64 4) ]
; CHECK-NEXT: ret ptr [[GEP]]
;
%l.1 = load ptr, ptr %p, align 8
%gep = getelementptr i8, ptr %l.1, i64 4
store ptr %gep, ptr %p, align 8
- %l.2 = load ptr, ptr %p, align 8, !align !0
+ %l.2 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.2
}
@@ -54,11 +74,12 @@ define ptr @align_replacement_has_smaller_alignment(ptr noalias %p) {
; CHECK-SAME: ptr noalias [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]]
; CHECK-NEXT: call void @foo(ptr [[L_1]])
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_1]], i64 8) ]
; CHECK-NEXT: ret ptr [[L_1]]
;
%l.1 = load ptr, ptr %p, align 8, !align !0
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !1
+ %l.2 = load ptr, ptr %p, align 8, !align !1, !noundef !{}
ret ptr %l.2
}
@@ -67,12 +88,12 @@ define ptr @align_replacement_has_larger_alignment(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8, !align [[META1:![0-9]+]]
; CHECK-NEXT: call void @foo(ptr [[L_1]])
-; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]]
+; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]], !noundef [[META2:![0-9]+]]
; CHECK-NEXT: ret ptr [[L_2]]
;
%l.1 = load ptr, ptr %p, align 8, !align !1
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !0
+ %l.2 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.2
}
@@ -81,12 +102,12 @@ define ptr @align_1(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT: call void @foo(ptr [[L_1]])
-; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META2:![0-9]+]]
+; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META3:![0-9]+]], !noundef [[META2]]
; CHECK-NEXT: ret ptr [[L_2]]
;
%l.1 = load ptr, ptr %p, align 8
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !2
+ %l.2 = load ptr, ptr %p, align 8, !align !2, !noundef !{}
ret ptr %l.2
}
@@ -96,5 +117,6 @@ define ptr @align_1(ptr %p) {
;.
; CHECK: [[META0]] = !{i64 4}
; CHECK: [[META1]] = !{i64 8}
-; CHECK: [[META2]] = !{i64 1}
+; CHECK: [[META2]] = !{}
+; CHECK: [[META3]] = !{i64 1}
;.
More information about the llvm-commits
mailing list