[llvm] [InstCombine] Fold align assume into load's !align metadata if possible. (PR #123247)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Jan 18 13:25:09 PST 2025
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/123247
>From aa9a17069aa98eb193a1ff13398d6df12c3863f0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Sep 2024 10:50:09 +0100
Subject: [PATCH 1/8] [InstCombine] Remove redundant alignment assumptions.
Use known bits to remove redundant alignment assumptions.
Libc++ now adds alignment assumptions for std::vector::begin() and
std::vector::end(), so I expect we will see quite a bit more
assumptions in C++ [1]. Try to clean up some redundant ones to start
with.
[1] https://github.com/llvm/llvm-project/pull/108961
---
.../InstCombine/InstCombineCalls.cpp | 31 ++++++++++++++++---
.../Transforms/InstCombine/assume-align.ll | 5 +--
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 842881156dc67f..ab628e94235913 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3199,12 +3199,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
// TODO: apply range metadata for range check patterns?
}
- // Separate storage assumptions apply to the underlying allocations, not any
- // particular pointer within them. When evaluating the hints for AA purposes
- // we getUnderlyingObject them; by precomputing the answers here we can
- // avoid having to do so repeatedly there.
for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
OperandBundleUse OBU = II->getOperandBundleAt(Idx);
+
+ // Separate storage assumptions apply to the underlying allocations, not
+ // any particular pointer within them. When evaluating the hints for AA
+ // purposes we getUnderlyingObject them; by precomputing the answers here
+ // we can avoid having to do so repeatedly there.
if (OBU.getTagName() == "separate_storage") {
assert(OBU.Inputs.size() == 2);
auto MaybeSimplifyHint = [&](const Use &U) {
@@ -3218,6 +3219,28 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MaybeSimplifyHint(OBU.Inputs[0]);
MaybeSimplifyHint(OBU.Inputs[1]);
}
+
+ // Try to remove redundant alignment assumptions.
+ if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
+ RetainedKnowledge RK = getKnowledgeFromBundle(
+ *cast<AssumeInst>(II), II->bundle_op_info_begin()[Idx]);
+ if (!RK || RK.AttrKind != Attribute::Alignment ||
+ !isPowerOf2_64(RK.ArgValue))
+ continue;
+
+ // Try to get the instruction before the assumption to use as context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (KnownAlign < RK.ArgValue)
+ continue;
+
+ auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
+ return New;
+ }
}
// Convert nonnull assume like:
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index f0e02574330861..962ad4f792a0d1 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -175,7 +175,6 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt
define ptr @redundant_assume_align_1(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_1(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 1) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -189,7 +188,6 @@ define ptr @redundant_assume_align_1(ptr %p) {
define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]]
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -214,8 +212,7 @@ define ptr @assume_align_16_via_align_metadata(ptr %p) {
define ptr @redundant_assume_align_8_via_align_attribute(ptr align 8 %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_attribute(
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P:%.*]], i32 8) ]
-; CHECK-NEXT: call void @foo(ptr [[P]])
+; CHECK-NEXT: call void @foo(ptr [[P:%.*]])
; CHECK-NEXT: ret ptr [[P]]
;
call void @llvm.assume(i1 true) [ "align"(ptr %p, i32 8) ]
>From f80ba163c7a43ed3fa4b3a715bef285c6d4a5757 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 17 Sep 2024 10:50:09 +0100
Subject: [PATCH 2/8] [InstCombine] Fold align assume into load's !align
metadata if possible.
If an alignment assumption is valid in the context of a corresponding
load of the pointer the assumption applies to, the assumption can be
replaced !align metadata on the load.
The benefits of folding it into !align are that existing code makes
better use of !align and it allows removing the now-redundant call
instructions.
!fix formatting.
!fixup check alignment is ConstantInt.
!fixup check align type size.
!fixup use getKnowledgeFromBundle
!fixup update test check
!fixup skip arguments early
---
.../InstCombine/InstCombineCalls.cpp | 31 ++++++++++++++-----
.../Transforms/InstCombine/assume-align.ll | 27 +++++++++++++++-
2 files changed, 49 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ab628e94235913..1ca01f75897d5a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3220,7 +3220,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MaybeSimplifyHint(OBU.Inputs[1]);
}
- // Try to remove redundant alignment assumptions.
+ // Try to fold alignment assumption into a load's !align metadata, if the
+ // assumption is valid in the load's context and remove redundant ones.
if (OBU.getTagName() == "align" && OBU.Inputs.size() == 2) {
RetainedKnowledge RK = getKnowledgeFromBundle(
*cast<AssumeInst>(II), II->bundle_op_info_begin()[Idx]);
@@ -3228,16 +3229,30 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
!isPowerOf2_64(RK.ArgValue))
continue;
- // Try to get the instruction before the assumption to use as context.
- Instruction *CtxI = nullptr;
- if (CtxI && II->getParent()->begin() != II->getIterator())
- CtxI = II->getPrevNode();
+ auto *LI = dyn_cast<LoadInst>(OBU.Inputs[0]);
+ if (LI &&
+ isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
- unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
- if (KnownAlign < RK.ArgValue)
+ if (isa<Argument>(LI->getPointerOperand()))
continue;
+ LI->setMetadata(
+ LLVMContext::MD_align,
+ MDNode::get(II->getContext(), ValueAsMetadata::getConstant(
+ Builder.getInt64(RK.ArgValue))));
+ MDNode *MD = MDNode::get(II->getContext(), {});
+ LI->setMetadata(LLVMContext::MD_noundef, MD);
+ } else {
+ // Try to get the instruction before the assumption to use as context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (KnownAlign < RK.ArgValue)
+ continue;
+ }
auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
return New;
}
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index 962ad4f792a0d1..cd7f5841769f25 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -123,7 +123,6 @@ define i8 @assume_align_non_pow2(ptr %p) {
ret i8 %v
}
-; TODO: Can fold alignment assumption into !align metadata on load.
define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
@@ -135,6 +134,17 @@ define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p)
ret ptr %p2
}
+define ptr @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
+; CHECK-LABEL: @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
+; CHECK-NEXT: ret ptr [[P2]]
+;
+ %p2 = load ptr, ptr %p
+ call void @llvm.assume(i1 true) [ "align"(ptr %p2, i32 8) ]
+ ret ptr %p2
+}
+
define ptr @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(ptr %p) {
; CHECK-LABEL: @dont_fold_assume_align_pow2_of_loaded_pointer_into_align_metadata_due_to_call(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
@@ -175,6 +185,7 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt
define ptr @redundant_assume_align_1(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_1(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 1) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -188,6 +199,7 @@ define ptr @redundant_assume_align_1(ptr %p) {
define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata(
; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]]
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -247,6 +259,19 @@ define ptr @redundant_assume_align_8_via_asume(ptr %p) {
}
declare void @foo(ptr)
+
+; !align must have a constant integer alignment.
+define ptr @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(ptr %p, i64 %align) {
+; CHECK-LABEL: @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 [[ALIGN:%.*]]) ]
+; CHECK-NEXT: ret ptr [[P2]]
+;
+ %p2 = load ptr, ptr %p
+ call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 %align) ]
+ ret ptr %p2
+}
+
;.
; CHECK: [[META0]] = !{i64 8}
;.
>From 6213d1ccee6770dec73d1a4b324fb732b2ecf35c Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 22 Dec 2024 11:16:58 +0000
Subject: [PATCH 3/8] [InstCombine] Add option to clean up some assumptions.
---
.../llvm/Transforms/InstCombine/InstCombine.h | 6 ++++++
llvm/lib/Passes/PassBuilder.cpp | 2 ++
llvm/lib/Passes/PassBuilderPipelines.cpp | 18 ++++++++++++------
llvm/lib/Passes/PassRegistry.def | 2 +-
.../InstCombine/InstCombineCalls.cpp | 3 +--
.../InstCombine/InstCombineInternal.h | 8 ++++++--
.../InstCombine/InstructionCombining.cpp | 4 ++--
7 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
index c12d749709cd25..dd9d05cbbcf7c9 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombine.h
@@ -31,6 +31,7 @@ struct InstCombineOptions {
// Verify that a fix point has been reached after MaxIterations.
bool VerifyFixpoint = false;
unsigned MaxIterations = InstCombineDefaultMaxIterations;
+ bool CleanupAssumptions = false;
InstCombineOptions() = default;
@@ -43,6 +44,11 @@ struct InstCombineOptions {
MaxIterations = Value;
return *this;
}
+
+ InstCombineOptions &setCleanupAssumptions(bool Value) {
+ CleanupAssumptions = Value;
+ return *this;
+ }
};
class InstCombinePass : public PassInfoMixin<InstCombinePass> {
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f698a3df08ef78..ebf91c2ff1ca90 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -918,6 +918,8 @@ Expected<InstCombineOptions> parseInstCombineOptions(StringRef Params) {
ParamName).str(),
inconvertibleErrorCode());
Result.setMaxIterations((unsigned)MaxIterations.getZExtValue());
+ } else if (ParamName == "cleanup-assumptions") {
+ Result.setCleanupAssumptions(Enable);
} else {
return make_error<StringError>(
formatv("invalid InstCombine pass parameter '{0}' ", ParamName).str(),
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 4ec0fb8fc81ea4..9985b946a54833 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -1305,7 +1305,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(LoopLoadEliminationPass());
}
// Cleanup after the loop optimization passes.
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
if (Level.getSpeedupLevel() > 1 && ExtraVectorizerPasses) {
ExtraFunctionPassManager<ShouldRunExtraVectorPasses> ExtraPasses;
@@ -1317,7 +1318,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
// dead (or speculatable) control flows or more combining opportunities.
ExtraPasses.addPass(EarlyCSEPass());
ExtraPasses.addPass(CorrelatedValuePropagationPass());
- ExtraPasses.addPass(InstCombinePass());
+ ExtraPasses.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
LoopPassManager LPM;
LPM.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap,
/*AllowSpeculation=*/true));
@@ -1328,7 +1330,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
/*UseBlockFrequencyInfo=*/true));
ExtraPasses.addPass(
SimplifyCFGPass(SimplifyCFGOptions().convertSwitchRangeToICmp(true)));
- ExtraPasses.addPass(InstCombinePass());
+ ExtraPasses.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
FPM.addPass(std::move(ExtraPasses));
}
@@ -1351,7 +1354,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
if (IsFullLTO) {
FPM.addPass(SCCPPass());
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
FPM.addPass(BDCEPass());
}
@@ -1366,7 +1370,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
FPM.addPass(VectorCombinePass());
if (!IsFullLTO) {
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
// Unroll small loops to hide loop backedge latency and saturate any
// parallel execution resources of an out-of-order processor. We also then
// need to clean up redundancies and loop invariant code.
@@ -1392,7 +1397,8 @@ void PassBuilder::addVectorPasses(OptimizationLevel Level,
}
FPM.addPass(InferAlignmentPass());
- FPM.addPass(InstCombinePass());
+ FPM.addPass(
+ InstCombinePass(InstCombineOptions().setCleanupAssumptions(true)));
// This is needed for two reasons:
// 1. It works around problems that instcombine introduces, such as sinking
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index a93a995655a147..fd985e46c71667 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -539,7 +539,7 @@ FUNCTION_PASS_WITH_PARAMS(
[](InstCombineOptions Opts) { return InstCombinePass(Opts); },
parseInstCombineOptions,
"no-use-loop-info;use-loop-info;no-verify-fixpoint;verify-fixpoint;"
- "max-iterations=N")
+ "max-iterations=N;cleanup-assumptions")
FUNCTION_PASS_WITH_PARAMS(
"loop-unroll", "LoopUnrollPass",
[](LoopUnrollOptions Opts) { return LoopUnrollPass(Opts); },
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 1ca01f75897d5a..700faf3e9b7e00 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3228,12 +3228,11 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (!RK || RK.AttrKind != Attribute::Alignment ||
!isPowerOf2_64(RK.ArgValue))
continue;
-
auto *LI = dyn_cast<LoadInst>(OBU.Inputs[0]);
if (LI &&
isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- if (isa<Argument>(LI->getPointerOperand()))
+ if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))
continue;
LI->setMetadata(
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 83e1da98deeda0..b365a41a87d7ac 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -59,6 +59,8 @@ class User;
class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
: public InstCombiner,
public InstVisitor<InstCombinerImpl, Instruction *> {
+ bool CleanupAssumptions = false;
+
public:
InstCombinerImpl(InstructionWorklist &Worklist, BuilderTy &Builder,
bool MinimizeSize, AAResults *AA, AssumptionCache &AC,
@@ -66,9 +68,11 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
DominatorTree &DT, OptimizationRemarkEmitter &ORE,
BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI,
ProfileSummaryInfo *PSI, const DataLayout &DL,
- ReversePostOrderTraversal<BasicBlock *> &RPOT)
+ ReversePostOrderTraversal<BasicBlock *> &RPOT,
+ bool CleanupAssumptions)
: InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE,
- BFI, BPI, PSI, DL, RPOT) {}
+ BFI, BPI, PSI, DL, RPOT),
+ CleanupAssumptions(CleanupAssumptions) {}
virtual ~InstCombinerImpl() = default;
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index fb21576722461e..9a3ff72281b963 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -5550,7 +5550,7 @@ static bool combineInstructionsOverFunction(
<< F.getName() << "\n");
InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT,
- ORE, BFI, BPI, PSI, DL, RPOT);
+ ORE, BFI, BPI, PSI, DL, RPOT, Opts.CleanupAssumptions);
IC.MaxArraySizeForCombine = MaxArraySize;
bool MadeChangeInThisIteration = IC.prepareWorklist(F);
MadeChangeInThisIteration |= IC.run();
@@ -5599,7 +5599,7 @@ PreservedAnalyses InstCombinePass::run(Function &F,
FunctionAnalysisManager &AM) {
auto &LRT = AM.getResult<LastRunTrackingAnalysis>(F);
// No changes since last InstCombine pass, exit early.
- if (LRT.shouldSkip(&ID))
+ if (LRT.shouldSkip(&ID) && !Options.CleanupAssumptions)
return PreservedAnalyses::all();
auto &AC = AM.getResult<AssumptionAnalysis>(F);
>From 32de5024662b1d38c91ad4737aac741faaa9dac5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 14:59:55 +0000
Subject: [PATCH 4/8] [Mem2Reg] Add align assumption
---
.../Utils/PromoteMemoryToRegister.cpp | 24 +++++++++++++++++++
.../AArch64/infer-align-from-assumption.ll | 14 ++++++-----
2 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 656bb1ebd1161e..a89471adb393c4 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -30,6 +30,7 @@
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/DebugInfo.h"
#include "llvm/IR/DebugProgramInstruction.h"
@@ -453,6 +454,15 @@ static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) {
AC->registerAssumption(cast<AssumeInst>(CI));
}
+
+static void addAssumeAlign(const DataLayout &DL, LoadInst *LI, Value *Val) {
+ auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+ IRBuilder Builder(LI);
+ Builder.CreateAlignmentAssumption(DL, Val, B);
+}
+
+
static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
const DataLayout &DL, AssumptionCache *AC,
const DominatorTree *DT) {
@@ -473,6 +483,20 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
LI->getMetadata(LLVMContext::MD_noundef) &&
!isKnownNonZero(Val, SimplifyQuery(DL, DT, AC, LI)))
addAssumeNonNull(AC, LI);
+
+ if (AC && LI->getMetadata(LLVMContext::MD_align) &&
+ LI->getMetadata(LLVMContext::MD_noundef)) {
+ auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+
+ auto KB = computeKnownBits(Val, 3, SimplifyQuery(DL, DT, AC, LI));
+ unsigned AlignFromKB = 1 << KB.countMinTrailingZeros();
+ if (AlignFromKB < B->getZExtValue()) {
+ addAssumeAlign(DL, LI, Val);
+}
+
+}
+
}
static void removeIntrinsicUsers(AllocaInst *AI) {
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
index 632e3a56aacac7..2440a61ccaf44f 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
@@ -8,8 +8,7 @@ declare void @llvm.assume(i1 noundef)
define i32 @earlycse_entry(ptr %p) {
; CHECK-LABEL: define i32 @earlycse_entry(
; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
-; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_I]], i64 4) ]
+; CHECK-NEXT: [[L_I:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0:![0-9]+]], !noundef [[META1:![0-9]+]]
; CHECK-NEXT: [[L_ASSUME_ALIGNED_I_I:%.*]] = load i32, ptr [[L_I]], align 4
; CHECK-NEXT: [[R_I_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I_I]])
; CHECK-NEXT: [[L_2_I:%.*]] = load ptr, ptr [[P]], align 8
@@ -31,8 +30,7 @@ define i32 @earlycse_entry(ptr %p) {
define i32 @earlycse_fn1(ptr %p) {
; CHECK-LABEL: define i32 @earlycse_fn1(
; CHECK-SAME: ptr nocapture [[P:%.*]]) local_unnamed_addr {
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 4) ]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]], !noundef [[META1]]
; CHECK-NEXT: [[L_ASSUME_ALIGNED_I:%.*]] = load i32, ptr [[L]], align 4
; CHECK-NEXT: [[R_I:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I]])
; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8
@@ -83,8 +81,7 @@ define void @sroa_align_entry(ptr %p) {
define ptr @sroa_fn1(ptr %p) {
; CHECK-LABEL: define ptr @sroa_fn1(
; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L]], i64 8) ]
+; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META2:![0-9]+]], !noundef [[META1]]
; CHECK-NEXT: [[L_FN3_I_I:%.*]] = load i64, ptr [[L]], align 8
; CHECK-NEXT: [[I_I:%.*]] = inttoptr i64 [[L_FN3_I_I]] to ptr
; CHECK-NEXT: ret ptr [[I_I]]
@@ -118,3 +115,8 @@ define i64 @sroa_fn3(ptr %0) {
%l.fn3 = load i64, ptr %0, align 1
ret i64 %l.fn3
}
+;.
+; CHECK: [[META0]] = !{i64 4}
+; CHECK: [[META1]] = !{}
+; CHECK: [[META2]] = !{i64 8}
+;.
>From 8838cb8fead8715a450951efe1aca356a218eeaa Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 15:00:55 +0000
Subject: [PATCH 5/8] DIsable
---
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 700faf3e9b7e00..de27ea1844e50d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3232,8 +3232,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
if (LI &&
isValidAssumeForContext(II, LI, &DT, /*AllowEphemerals=*/true)) {
- if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))
- continue;
+/* if (!CleanupAssumptions && isa<Argument>(LI->getPointerOperand()))*/
+ /*continue;*/
LI->setMetadata(
LLVMContext::MD_align,
>From fcb2fa119bd83068a41acfd37fb85d13daf7d8c8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jan 2025 16:36:49 +0000
Subject: [PATCH 6/8] Step
---
.../InstCombine/InstCombineCalls.cpp | 73 ++++++++++++++++---
llvm/lib/Transforms/Scalar/EarlyCSE.cpp | 21 ++++++
.../Utils/PromoteMemoryToRegister.cpp | 7 +-
.../EarlyCSE/materialize-align-assumptions.ll | 38 ++++++++--
4 files changed, 118 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index de27ea1844e50d..4d8e93437b9bd0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3242,15 +3242,70 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
MDNode *MD = MDNode::get(II->getContext(), {});
LI->setMetadata(LLVMContext::MD_noundef, MD);
} else {
- // Try to get the instruction before the assumption to use as context.
- Instruction *CtxI = nullptr;
- if (CtxI && II->getParent()->begin() != II->getIterator())
- CtxI = II->getPrevNode();
-
- auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
- unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
- if (KnownAlign < RK.ArgValue)
- continue;
+ auto *C = dyn_cast<Constant>(RK.WasOn);
+ if (C && C->isNullValue()) {
+ } else {
+ Value *UO = getUnderlyingObject(RK.WasOn);
+
+ bool CanUseAlign = false;
+ SetVector<const Instruction *> WorkList;
+ for (const User *U : RK.WasOn->users())
+ if (auto *I = dyn_cast<Instruction>(U))
+ WorkList.insert(I);
+
+ for (unsigned I = 0; I != WorkList.size(); ++I) {
+ auto *Curr = WorkList[I];
+ if (!DT.dominates(II, Curr))
+ continue;
+ if (auto *LI = dyn_cast<LoadInst>(Curr)) {
+ if (LI->getAlign().value() < RK.ArgValue) {
+ CanUseAlign = true;
+ break;
+ }
+ continue;
+ }
+ if (auto *SI = dyn_cast<StoreInst>(Curr)) {
+ auto *PtrOpI = dyn_cast<Instruction>(SI->getPointerOperand());
+ if (PtrOpI && WorkList.contains(PtrOpI) &&
+ SI->getAlign().value() < RK.ArgValue) {
+ CanUseAlign = true;
+ break;
+ }
+ continue;
+ }
+ if (isa<ReturnInst, CallBase>(Curr)) {
+ CanUseAlign = true;
+ break;
+ }
+ if (isa<ICmpInst>(Curr) &&
+ !isa<Constant>(cast<Instruction>(Curr)->getOperand(0)) &&
+ !isa<Constant>(cast<Instruction>(Curr)->getOperand(1))) {
+ CanUseAlign = true;
+ break;
+ }
+ if (!Curr->getType()->isPointerTy())
+ continue;
+
+ if (WorkList.size() > 16) {
+ CanUseAlign = true;
+ break;
+ }
+ for (const User *U : Curr->users())
+ WorkList.insert(cast<Instruction>(U));
+ }
+ if (CanUseAlign && (!UO || isa<Argument>(UO)))
+ continue;
+ // Try to get the instruction before the assumption to use as
+ // context.
+ Instruction *CtxI = nullptr;
+ if (CtxI && II->getParent()->begin() != II->getIterator())
+ CtxI = II->getPrevNode();
+
+ auto Known = computeKnownBits(RK.WasOn, 1, CtxI);
+ unsigned KnownAlign = 1 << Known.countMinTrailingZeros();
+ if (CanUseAlign && KnownAlign < RK.ArgValue)
+ continue;
+ }
}
auto *New = CallBase::removeOperandBundle(II, OBU.getTagID());
return New;
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 3a0ae6b01a1144..7ac1df8b3d0560 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -16,6 +16,7 @@
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopedHashTable.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
@@ -31,6 +32,7 @@
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
@@ -1599,6 +1601,25 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
if (InVal.IsLoad)
if (auto *I = dyn_cast<Instruction>(Op))
combineMetadataForCSE(I, &Inst, false);
+
+ // If the load has align and noundef metadata, preserve it via an
+ // alignment assumption. Note that this doesn't use salavageKnowledge,
+ // as we need to create the assumption for the value we replaced the
+ // load with.
+ if (auto *AlignMD = Inst.getMetadata(LLVMContext::MD_align)) {
+ if (Inst.hasMetadata(LLVMContext::MD_noundef) ||
+ programUndefinedIfPoison(&Inst)) {
+ Inst.setMetadata(LLVMContext::MD_align, nullptr);
+ auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
+ auto KB = computeKnownBits(Op, 3, SQ.getWithInstruction(&Inst));
+ unsigned AlignFromKB = 1 << KB.countMinTrailingZeros();
+ if (AlignFromKB < B->getZExtValue()) {
+ IRBuilder Builder(&Inst);
+ Builder.CreateAlignmentAssumption(SQ.DL, Op, B);
+ }
+ }
+ }
+
if (!Inst.use_empty())
Inst.replaceAllUsesWith(Op);
salvageKnowledge(&Inst, &AC);
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index a89471adb393c4..93d9767522bb51 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -485,7 +485,8 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
addAssumeNonNull(AC, LI);
if (AC && LI->getMetadata(LLVMContext::MD_align) &&
- LI->getMetadata(LLVMContext::MD_noundef)) {
+ (LI->getMetadata(LLVMContext::MD_noundef) ||
+ programUndefinedIfPoison(LI))) {
auto *AlignMD = LI->getMetadata(LLVMContext::MD_align);
auto *B = mdconst::extract<ConstantInt>(AlignMD->getOperand(0));
@@ -494,9 +495,7 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
if (AlignFromKB < B->getZExtValue()) {
addAssumeAlign(DL, LI, Val);
}
-
-}
-
+ }
}
static void removeIntrinsicUsers(AllocaInst *AI) {
diff --git a/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll b/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
index ea63376957162b..837a73a00d6431 100644
--- a/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
+++ b/llvm/test/Transforms/EarlyCSE/materialize-align-assumptions.ll
@@ -3,6 +3,24 @@
declare void @foo(ptr)
+define ptr @align_replacement_does_not_have_align_metadata_missing_noundef(ptr noalias %p) {
+; CHECK-LABEL: define ptr @align_replacement_does_not_have_align_metadata_missing_noundef(
+; CHECK-SAME: ptr noalias [[P:%.*]]) {
+; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
+; CHECK-NEXT: call void @foo(ptr [[L_1]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
+; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: ret ptr [[GEP]]
+;
+ %l.1 = load ptr, ptr %p, align 8
+ call void @foo(ptr %l.1)
+ %l.2 = load ptr, ptr %p, align 8
+ %gep = getelementptr i8, ptr %l.2, i64 4
+ store ptr %gep, ptr %p, align 8
+ %l.3 = load ptr, ptr %p, align 8, !align !0
+ ret ptr %l.3
+}
+
define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
; CHECK-LABEL: define ptr @align_replacement_does_not_have_align_metadata(
; CHECK-SAME: ptr noalias [[P:%.*]]) {
@@ -10,6 +28,7 @@ define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
; CHECK-NEXT: call void @foo(ptr [[L_1]])
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[GEP]], i64 4) ]
; CHECK-NEXT: ret ptr [[GEP]]
;
%l.1 = load ptr, ptr %p, align 8
@@ -17,7 +36,7 @@ define ptr @align_replacement_does_not_have_align_metadata(ptr noalias %p) {
%l.2 = load ptr, ptr %p, align 8
%gep = getelementptr i8, ptr %l.2, i64 4
store ptr %gep, ptr %p, align 8
- %l.3 = load ptr, ptr %p, align 8, !align !0
+ %l.3 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.3
}
@@ -27,12 +46,13 @@ define ptr @align_replacement_does_not_have_align_metadata2(ptr noalias %p) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[L_1]], i64 4
; CHECK-NEXT: store ptr [[GEP]], ptr [[P]], align 8
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[GEP]], i64 4) ]
; CHECK-NEXT: ret ptr [[GEP]]
;
%l.1 = load ptr, ptr %p, align 8
%gep = getelementptr i8, ptr %l.1, i64 4
store ptr %gep, ptr %p, align 8
- %l.2 = load ptr, ptr %p, align 8, !align !0
+ %l.2 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.2
}
@@ -54,11 +74,12 @@ define ptr @align_replacement_has_smaller_alignment(ptr noalias %p) {
; CHECK-SAME: ptr noalias [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]]
; CHECK-NEXT: call void @foo(ptr [[L_1]])
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[L_1]], i64 8) ]
; CHECK-NEXT: ret ptr [[L_1]]
;
%l.1 = load ptr, ptr %p, align 8, !align !0
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !1
+ %l.2 = load ptr, ptr %p, align 8, !align !1, !noundef !{}
ret ptr %l.2
}
@@ -67,12 +88,12 @@ define ptr @align_replacement_has_larger_alignment(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8, !align [[META1:![0-9]+]]
; CHECK-NEXT: call void @foo(ptr [[L_1]])
-; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]]
+; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META0]], !noundef [[META2:![0-9]+]]
; CHECK-NEXT: ret ptr [[L_2]]
;
%l.1 = load ptr, ptr %p, align 8, !align !1
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !0
+ %l.2 = load ptr, ptr %p, align 8, !align !0, !noundef !{}
ret ptr %l.2
}
@@ -81,12 +102,12 @@ define ptr @align_1(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[L_1:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT: call void @foo(ptr [[L_1]])
-; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META2:![0-9]+]]
+; CHECK-NEXT: [[L_2:%.*]] = load ptr, ptr [[P]], align 8, !align [[META3:![0-9]+]], !noundef [[META2]]
; CHECK-NEXT: ret ptr [[L_2]]
;
%l.1 = load ptr, ptr %p, align 8
call void @foo(ptr %l.1)
- %l.2 = load ptr, ptr %p, align 8, !align !2
+ %l.2 = load ptr, ptr %p, align 8, !align !2, !noundef !{}
ret ptr %l.2
}
@@ -96,5 +117,6 @@ define ptr @align_1(ptr %p) {
;.
; CHECK: [[META0]] = !{i64 4}
; CHECK: [[META1]] = !{i64 8}
-; CHECK: [[META2]] = !{i64 1}
+; CHECK: [[META2]] = !{}
+; CHECK: [[META3]] = !{i64 1}
;.
>From 0778e2e24385af74c0ad0f923b4080cdd707d9f9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 18 Jan 2025 20:29:00 +0000
Subject: [PATCH 7/8] Test update
---
.../Transforms/InstCombine/assume-align.ll | 55 ++++++++++++++-----
.../AArch64/infer-align-from-assumption.ll | 26 ++++++---
2 files changed, 57 insertions(+), 24 deletions(-)
diff --git a/llvm/test/Transforms/InstCombine/assume-align.ll b/llvm/test/Transforms/InstCombine/assume-align.ll
index cd7f5841769f25..4df2e865b94931 100644
--- a/llvm/test/Transforms/InstCombine/assume-align.ll
+++ b/llvm/test/Transforms/InstCombine/assume-align.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals smart
-; RUN: opt -S -passes=instcombine,simplifycfg < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='instcombine<no-verify-fixpoint>,simplifycfg' < %s 2>&1 | FileCheck %s
declare void @llvm.assume(i1 noundef)
@@ -87,7 +87,6 @@ if.end: ; preds = %if.else, %if.then
define void @f3(i64 %a, ptr %b) {
; CHECK-LABEL: @f3(
; CHECK-NEXT: [[C:%.*]] = ptrtoint ptr [[B:%.*]] to i64
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[B]], i64 4294967296) ]
; CHECK-NEXT: [[D:%.*]] = add i64 [[A:%.*]], [[C]]
; CHECK-NEXT: call void @g(i64 [[D]])
; CHECK-NEXT: ret void
@@ -125,8 +124,7 @@ define i8 @assume_align_non_pow2(ptr %p) {
define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
; CHECK-LABEL: @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 8) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]], !noundef [[META1:![0-9]+]]
; CHECK-NEXT: ret ptr [[P2]]
;
%p2 = load ptr, ptr %p
@@ -136,8 +134,7 @@ define ptr @fold_assume_align_pow2_of_loaded_pointer_into_align_metadata(ptr %p)
define ptr @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(ptr %p) {
; CHECK-LABEL: @fold_assume_align_i32_pow2_of_loaded_pointer_into_align_metadata(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]], !noundef [[META1]]
; CHECK-NEXT: ret ptr [[P2]]
;
%p2 = load ptr, ptr %p
@@ -184,8 +181,7 @@ define ptr @dont_fold_assume_align_zero_of_loaded_pointer_into_align_metadata(pt
define ptr @redundant_assume_align_1(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_1(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 1) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META2:![0-9]+]], !noundef [[META1]]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -198,8 +194,7 @@ define ptr @redundant_assume_align_1(ptr %p) {
define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_metadata(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0:![0-9]+]]
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 8) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]], !noundef [[META1]]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -211,8 +206,7 @@ define ptr @redundant_assume_align_8_via_align_metadata(ptr %p) {
define ptr @assume_align_16_via_align_metadata(ptr %p) {
; CHECK-LABEL: @assume_align_16_via_align_metadata(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META0]]
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i32 16) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META3:![0-9]+]], !noundef [[META1]]
; CHECK-NEXT: call void @foo(ptr [[P2]])
; CHECK-NEXT: ret ptr [[P2]]
;
@@ -224,7 +218,8 @@ define ptr @assume_align_16_via_align_metadata(ptr %p) {
define ptr @redundant_assume_align_8_via_align_attribute(ptr align 8 %p) {
; CHECK-LABEL: @redundant_assume_align_8_via_align_attribute(
-; CHECK-NEXT: call void @foo(ptr [[P:%.*]])
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P:%.*]], i32 8) ]
+; CHECK-NEXT: call void @foo(ptr [[P]])
; CHECK-NEXT: ret ptr [[P]]
;
call void @llvm.assume(i1 true) [ "align"(ptr %p, i32 8) ]
@@ -258,13 +253,40 @@ define ptr @redundant_assume_align_8_via_asume(ptr %p) {
ret ptr %p
}
+define void @redundant_arg_passed_to_intrinsic(ptr %dst, ptr %src) {
+; CHECK-LABEL: @redundant_arg_passed_to_intrinsic(
+; CHECK-NEXT: call void @bar()
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[SRC:%.*]], i32 8) ]
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) [[DST:%.*]], ptr noundef nonnull align 8 dereferenceable(16) [[SRC]], i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ call void @llvm.assume(i1 true) [ "align"(ptr %dst, i32 8) ]
+ call void @bar()
+ call void @llvm.assume(i1 true) [ "align"(ptr %src, i32 8) ]
+ call void @llvm.memmove.p0.p0.i64(ptr align 8 %dst, ptr %src, i64 16, i1 false)
+ ret void
+}
+
+define void @test_store(ptr %ptr) {
+; CHECK-LABEL: @test_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PTR:%.*]], i64 2) ]
+; CHECK-NEXT: store i16 0, ptr [[PTR]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ call void @llvm.assume(i1 true) [ "align"(ptr %ptr, i64 2) ]
+ store i16 0, ptr %ptr, align 1
+ ret void
+}
+
declare void @foo(ptr)
+declare void @bar()
; !align must have a constant integer alignment.
define ptr @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(ptr %p, i64 %align) {
; CHECK-LABEL: @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_metadata(
-; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 [[ALIGN:%.*]]) ]
+; CHECK-NEXT: [[P2:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META2]], !noundef [[META1]]
; CHECK-NEXT: ret ptr [[P2]]
;
%p2 = load ptr, ptr %p
@@ -274,4 +296,7 @@ define ptr @dont_fold_assume_align_not_constant_of_loaded_pointer_into_align_met
;.
; CHECK: [[META0]] = !{i64 8}
+; CHECK: [[META1]] = !{}
+; CHECK: [[META2]] = !{i64 1}
+; CHECK: [[META3]] = !{i64 16}
;.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
index 2440a61ccaf44f..c7220ecf6ada50 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/infer-align-from-assumption.ll
@@ -14,7 +14,6 @@ define i32 @earlycse_entry(ptr %p) {
; CHECK-NEXT: [[L_2_I:%.*]] = load ptr, ptr [[P]], align 8
; CHECK-NEXT: [[GEP_I:%.*]] = getelementptr i8, ptr [[L_2_I]], i64 4
; CHECK-NEXT: store ptr [[GEP_I]], ptr [[P]], align 8
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[GEP_I]], i64 4) ]
; CHECK-NEXT: [[L_ASSUME_ALIGNED_I_I2:%.*]] = load i32, ptr [[GEP_I]], align 4
; CHECK-NEXT: [[R_I_I3:%.*]] = tail call i32 @swap(i32 [[L_ASSUME_ALIGNED_I_I2]])
; CHECK-NEXT: [[L_2_I4:%.*]] = load ptr, ptr [[P]], align 8
@@ -49,7 +48,6 @@ define i32 @earlycse_fn1(ptr %p) {
define i32 @load_assume_aligned(ptr %p) {
; CHECK-LABEL: define i32 @load_assume_aligned(
; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr {
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P]], i64 4) ]
; CHECK-NEXT: [[DOT0_COPYLOAD:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = tail call i32 @swap(i32 [[DOT0_COPYLOAD]])
; CHECK-NEXT: ret i32 [[TMP2]]
@@ -64,8 +62,7 @@ declare i32 @swap(i32)
define void @sroa_align_entry(ptr %p) {
; CHECK-LABEL: define void @sroa_align_entry(
-; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P]], i64 8) ]
+; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[DOT0_COPYLOAD_I_I_I:%.*]] = load i64, ptr [[P]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[DOT0_COPYLOAD_I_I_I]] to ptr
; CHECK-NEXT: store i32 0, ptr [[TMP2]], align 4
@@ -80,7 +77,7 @@ define void @sroa_align_entry(ptr %p) {
define ptr @sroa_fn1(ptr %p) {
; CHECK-LABEL: define ptr @sroa_fn1(
-; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
+; CHECK-SAME: ptr nocapture readonly [[P:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[P]], align 8, !align [[META2:![0-9]+]], !noundef [[META1]]
; CHECK-NEXT: [[L_FN3_I_I:%.*]] = load i64, ptr [[L]], align 8
; CHECK-NEXT: [[I_I:%.*]] = inttoptr i64 [[L_FN3_I_I]] to ptr
@@ -93,8 +90,7 @@ define ptr @sroa_fn1(ptr %p) {
define ptr @sroa_fn2(ptr %p) {
; CHECK-LABEL: define ptr @sroa_fn2(
-; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P]], i64 8) ]
+; CHECK-SAME: ptr [[P:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: [[DOT0_COPYLOAD_I_I:%.*]] = load i64, ptr [[P]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[DOT0_COPYLOAD_I_I]] to ptr
; CHECK-NEXT: ret ptr [[TMP3]]
@@ -106,8 +102,7 @@ define ptr @sroa_fn2(ptr %p) {
define i64 @sroa_fn3(ptr %0) {
; CHECK-LABEL: define i64 @sroa_fn3(
-; CHECK-SAME: ptr [[TMP0:%.*]]) local_unnamed_addr #[[ATTR3]] {
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[TMP0]], i64 8) ]
+; CHECK-SAME: ptr [[TMP0:%.*]]) local_unnamed_addr #[[ATTR2]] {
; CHECK-NEXT: [[DOT0_COPYLOAD_I:%.*]] = load i64, ptr [[TMP0]], align 8
; CHECK-NEXT: ret i64 [[DOT0_COPYLOAD_I]]
;
@@ -115,6 +110,19 @@ define i64 @sroa_fn3(ptr %0) {
%l.fn3 = load i64, ptr %0, align 1
ret i64 %l.fn3
}
+
+define void @test_store(ptr %ptr) {
+; CHECK-LABEL: define void @test_store(
+; CHECK-SAME: ptr [[PTR:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: store i16 0, ptr [[PTR]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ call void @llvm.assume(i1 true) [ "align"(ptr %ptr, i64 2) ]
+ store i16 0, ptr %ptr, align 1
+ ret void
+}
;.
; CHECK: [[META0]] = !{i64 4}
; CHECK: [[META1]] = !{}
>From 07c4290fefce0a415635ca3a6e0846b2c3358be4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 18 Jan 2025 21:24:24 +0000
Subject: [PATCH 8/8] Skip for Intrinsics and correct poitners
---
.../Transforms/InstCombine/InstCombineCalls.cpp | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 4d8e93437b9bd0..b5813bc47513ea 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3266,13 +3266,26 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
}
if (auto *SI = dyn_cast<StoreInst>(Curr)) {
auto *PtrOpI = dyn_cast<Instruction>(SI->getPointerOperand());
- if (PtrOpI && WorkList.contains(PtrOpI) &&
+ if ((SI->getPointerOperand() == RK.WasOn || (PtrOpI && WorkList.contains(PtrOpI))) &&
SI->getAlign().value() < RK.ArgValue) {
CanUseAlign = true;
break;
}
continue;
}
+ if (auto *II = dyn_cast<IntrinsicInst>(Curr)) {
+ for (const auto &[Idx, Arg] : enumerate(II->args())) {
+ if (Arg != RK.WasOn)
+ continue;
+ if (II->getParamAlign(Idx) >= RK.ArgValue)
+ continue;
+ CanUseAlign = true;
+ break;
+ }
+ if (CanUseAlign)
+ break;
+ continue;
+ }
if (isa<ReturnInst, CallBase>(Curr)) {
CanUseAlign = true;
break;
More information about the llvm-commits
mailing list