[llvm] 1b46cad - [LICM] Drop per-iteration AA tags (#223530)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 5 09:50:55 PDT 2026
Author: Zach Goldthorpe
Date: 2026-10-05T10:50:41-06:00
New Revision: 1b46cade4d774d80a936a9c15f363288ba34e27e
URL: https://github.com/llvm/llvm-project/commit/1b46cade4d774d80a936a9c15f363288ba34e27e
DIFF: https://github.com/llvm/llvm-project/commit/1b46cade4d774d80a936a9c15f363288ba34e27e.diff
LOG: [LICM] Drop per-iteration AA tags (#223530)
AA tags are scoped inside the loop via
`llvm.experimental.noalias.scope.decl` were being preserved in #222686
to determine promotion, which is incorrect as the alias metadata only
holds per individual loop iteration, and thus cannot be used to infer
alias information of memory locations used between iterations.
This patch checks for loop-local alias declarations and excludes them
from being preserved.
Added:
Modified:
llvm/lib/Transforms/Scalar/LICM.cpp
llvm/test/Transforms/LICM/scalar-promote-aa-tags.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 4cbcd3da56311..aa8bba03f5ce5 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -212,10 +212,10 @@ static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
function_ref<void(Instruction *)> Fn);
using PointersAndHasReadsOutsideSet =
std::pair<SmallSetVector<Value *, 8>, bool>;
-static SmallVector<PointersAndHasReadsOutsideSet, 0>
-collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA,
- DominatorTree *DT, ICFLoopSafetyInfo *SafetyInfo,
- Loop *L);
+static SmallVector<PointersAndHasReadsOutsideSet, 0> collectPromotionCandidates(
+ MemorySSA *MSSA, AliasAnalysis *AA, DominatorTree *DT,
+ ICFLoopSafetyInfo *SafetyInfo,
+ const SmallPtrSetImpl<const MDNode *> &LoopLocalAliasScopes, Loop *L);
namespace {
struct LoopInvariantCodeMotion {
@@ -433,11 +433,22 @@ bool LoopInvariantCodeMotion::runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI,
// there is currently no general solution for this. Similar issues could also
// potentially happen in other passes where instructions are being moved
// across that edge.
- bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
- using namespace PatternMatch;
- return any_of(make_pointer_range(*BB),
- match_fn(m_Intrinsic<Intrinsic::coro_suspend>()));
- });
+ bool HasCoroSuspendInst = false;
+
+ // AA metadata declared to be local to each iteration cannot be used to infer
+ // alias information when promoting stores.
+ SmallPtrSet<const MDNode *, 4> LoopLocalAliasScopes;
+
+ for (BasicBlock *BB : L->getBlocks()) {
+ for (Instruction &I : *BB) {
+ using namespace PatternMatch;
+ HasCoroSuspendInst |= match(&I, m_Intrinsic<Intrinsic::coro_suspend>());
+
+ if (auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&I))
+ for (const MDOperand &Op : Decl->getScopeList()->operands())
+ LoopLocalAliasScopes.insert(cast<MDNode>(Op.get()));
+ }
+ }
MemorySSAUpdater MSSAU(MSSA);
SinkAndHoistLICMFlags Flags(LicmMssaOptCap, LicmMssaNoAccForPromotionCap,
@@ -508,7 +519,8 @@ bool LoopInvariantCodeMotion::runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI,
do {
LocalPromoted = false;
for (auto [PointerMustAliases, HasReadsOutsideSet] :
- collectPromotionCandidates(MSSA, AA, DT, &SafetyInfo, L)) {
+ collectPromotionCandidates(MSSA, AA, DT, &SafetyInfo,
+ LoopLocalAliasScopes, L)) {
LocalPromoted |= promoteLoopAccessesToScalars(
PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
DT, AC, TLI, TTI, L, MSSAU, &SafetyInfo, ORE,
@@ -2035,11 +2047,21 @@ static bool isPotentiallyPromotable(const Instruction *I, const Loop *L) {
return false;
}
+/// Returns whether \p N has any operand from the set \p Operands.
+static bool
+hasAnyMDOperandsFrom(const MDNode *N,
+ const SmallPtrSetImpl<const MDNode *> &Operands) {
+ return N && llvm::any_of(N->operands(), [&](const MDOperand &Op) {
+ return Operands.contains(cast<MDNode>(Op.get()));
+ });
+}
+
/// Returns the potentially promotable stores with AA tags that are valid along
/// all non-unwinding execution paths of the loop \p L, which allows for the AA
/// tags to be used when deciding promotions.
-static SmallPtrSet<const StoreInst *, 8>
-collectStoresWithInvariantAATags(MemorySSA *MSSA, DominatorTree *DT, Loop *L) {
+static SmallPtrSet<const StoreInst *, 8> collectStoresWithInvariantAATags(
+ MemorySSA *MSSA, DominatorTree *DT,
+ const SmallPtrSetImpl<const MDNode *> &LoopLocalAliasScopes, Loop *L) {
SmallDenseMap<MemoryLocation, SmallVector<const StoreInst *, 1>, 4>
StoresByLoc;
foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
@@ -2054,7 +2076,16 @@ collectStoresWithInvariantAATags(MemorySSA *MSSA, DominatorTree *DT, Loop *L) {
L->getExitingBlocks(ExitingBlocks);
SmallPtrSet<const StoreInst *, 8> StoresWithInvariantAATags;
- for (const auto &Stores : llvm::make_second_range(StoresByLoc)) {
+ for (const auto &Pair : StoresByLoc) {
+ const MemoryLocation &Loc = Pair.first;
+ const SmallVector<const StoreInst *, 1> &Stores = Pair.second;
+
+ // A scope declared inside the loop denotes a
diff erent scope on each
+ // iteration, and thus should not be preserved.
+ if (hasAnyMDOperandsFrom(Loc.AATags.Scope, LoopLocalAliasScopes) ||
+ hasAnyMDOperandsFrom(Loc.AATags.NoAlias, LoopLocalAliasScopes))
+ continue;
+
// Without exiting blocks the loop is never left, and promotion has no
// exit block to insert a store into either.
if (llvm::all_of(ExitingBlocks, [&](BasicBlock *ExitingBB) {
@@ -2069,10 +2100,10 @@ collectStoresWithInvariantAATags(MemorySSA *MSSA, DominatorTree *DT, Loop *L) {
// The bool indicates whether there might be reads outside the set, in which
// case only loads may be promoted.
-static SmallVector<PointersAndHasReadsOutsideSet, 0>
-collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA,
- DominatorTree *DT, ICFLoopSafetyInfo *SafetyInfo,
- Loop *L) {
+static SmallVector<PointersAndHasReadsOutsideSet, 0> collectPromotionCandidates(
+ MemorySSA *MSSA, AliasAnalysis *AA, DominatorTree *DT,
+ ICFLoopSafetyInfo *SafetyInfo,
+ const SmallPtrSetImpl<const MDNode *> &LoopLocalAliasScopes, Loop *L) {
BatchAAResults BatchAA(*AA);
AliasSetTracker AST(BatchAA);
@@ -2081,7 +2112,8 @@ collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA,
std::optional<SmallPtrSet<const StoreInst *, 8>> StoresWithInvariantAATags;
auto HasInvariantAATags = [&](const StoreInst *SI) {
if (!StoresWithInvariantAATags)
- StoresWithInvariantAATags = collectStoresWithInvariantAATags(MSSA, DT, L);
+ StoresWithInvariantAATags =
+ collectStoresWithInvariantAATags(MSSA, DT, LoopLocalAliasScopes, L);
return StoresWithInvariantAATags->contains(SI);
};
diff --git a/llvm/test/Transforms/LICM/scalar-promote-aa-tags.ll b/llvm/test/Transforms/LICM/scalar-promote-aa-tags.ll
index 4b81b826aeb1c..c7bd7265f1986 100644
--- a/llvm/test/Transforms/LICM/scalar-promote-aa-tags.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote-aa-tags.ll
@@ -414,11 +414,9 @@ define i32 @not_promotable.per_iteration_noalias_scope(i64 %idx, i1 %c, i1 %c2)
; CHECK-SAME: i64 [[IDX:%.*]], i1 [[C:%.*]], i1 [[C2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[PTR:%.*]] = alloca [4 x i32], align 4
-; CHECK-NEXT: [[PTR_PROMOTED:%.*]] = load i32, ptr [[PTR]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[V_INC1:%.*]] = phi i32 [ [[PTR_PROMOTED]], %[[ENTRY]] ], [ [[V_INC2:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IDX]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IDX]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8]])
; CHECK-NEXT: [[FPTR:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[IV]]
; CHECK-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
@@ -426,15 +424,14 @@ define i32 @not_promotable.per_iteration_noalias_scope(i64 %idx, i1 %c, i1 %c2)
; CHECK-NEXT: store i32 42, ptr [[FPTR]], align 4, !alias.scope [[META8]]
; CHECK-NEXT: br label %[[LATCH]]
; CHECK: [[ELSE]]:
+; CHECK-NEXT: [[V_INC1:%.*]] = load i32, ptr [[PTR]], align 4, !noalias [[META8]]
; CHECK-NEXT: [[V_INC:%.*]] = add i32 [[V_INC1]], 1
+; CHECK-NEXT: store i32 [[V_INC]], ptr [[PTR]], align 4, !noalias [[META8]]
; CHECK-NEXT: br i1 [[C2]], label %[[EXIT:.*]], label %[[LATCH]]
; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[V_INC2]] = phi i32 [ [[V_INC]], %[[ELSE]] ], [ [[V_INC1]], %[[IF]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[V_INC_LCSSA:%.*]] = phi i32 [ [[V_INC]], %[[ELSE]] ]
-; CHECK-NEXT: store i32 [[V_INC_LCSSA]], ptr [[PTR]], align 4
; CHECK-NEXT: [[RES:%.*]] = load i32, ptr [[PTR]], align 4
; CHECK-NEXT: ret i32 [[RES]]
;
@@ -530,11 +527,9 @@ define i32 @not_promotable.per_iteration_alias_scope(i64 %idx, i1 %c, i1 %c2) {
; CHECK-SAME: i64 [[IDX:%.*]], i1 [[C:%.*]], i1 [[C2:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[PTR:%.*]] = alloca [4 x i32], align 4
-; CHECK-NEXT: [[PTR_PROMOTED:%.*]] = load i32, ptr [[PTR]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[V:%.*]] = phi i32 [ [[PTR_PROMOTED]], %[[ENTRY]] ], [ [[V_INC1:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IDX]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IDX]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META8]])
; CHECK-NEXT: [[FPTR:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[IV]]
; CHECK-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
@@ -542,15 +537,14 @@ define i32 @not_promotable.per_iteration_alias_scope(i64 %idx, i1 %c, i1 %c2) {
; CHECK-NEXT: store i32 42, ptr [[FPTR]], align 4, !noalias [[META8]]
; CHECK-NEXT: br label %[[LATCH]]
; CHECK: [[ELSE]]:
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[PTR]], align 4, !alias.scope [[META8]]
; CHECK-NEXT: [[V_INC:%.*]] = add i32 [[V]], 1
+; CHECK-NEXT: store i32 [[V_INC]], ptr [[PTR]], align 4, !alias.scope [[META8]]
; CHECK-NEXT: br i1 [[C2]], label %[[EXIT:.*]], label %[[LATCH]]
; CHECK: [[LATCH]]:
-; CHECK-NEXT: [[V_INC1]] = phi i32 [ [[V_INC]], %[[ELSE]] ], [ [[V]], %[[IF]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: br label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[V_INC_LCSSA:%.*]] = phi i32 [ [[V_INC]], %[[ELSE]] ]
-; CHECK-NEXT: store i32 [[V_INC_LCSSA]], ptr [[PTR]], align 4
; CHECK-NEXT: [[RES:%.*]] = load i32, ptr [[PTR]], align 4
; CHECK-NEXT: ret i32 [[RES]]
;
More information about the llvm-commits
mailing list