[llvm] [LICM] Rewrite noConflictingReadWrites to walk MemorySSA graph (PR #191468)
Sebastian Pop via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 12:14:46 PDT 2026
https://github.com/sebpop updated https://github.com/llvm/llvm-project/pull/191468
>From 12ccf4988e7ab662cc647af59a012772676cf2cc Mon Sep 17 00:00:00 2001
From: Sebastian Pop <spop at nvidia.com>
Date: Fri, 10 Apr 2026 10:53:24 -0500
Subject: [PATCH] [LICM] Rewrite noConflictingReadWrites to walk MemorySSA
graph
Instead of scanning all memory accesses in the loop and checking each
one with a clobber walk, traverse the MemorySSA use-def graph starting
from the loop header's MemoryPhi. In optimized MemorySSA, non-aliasing
MemoryUses have their defining access redirected outside the loop, so
they naturally won't appear as users of loop-internal accesses.
This removes: the O(all accesses) block scan, the tooManyMemoryAccesses
bailout, the incorrect dominance check (PR #187529), and the FIXME
about imprecise alias checking for MemoryUses.
---
llvm/lib/Transforms/Scalar/LICM.cpp | 113 ++++++----
llvm/test/Analysis/MemorySSA/pr43427.ll | 16 +-
.../2011-04-06-PromoteResultOfPromotion.ll | 17 +-
llvm/test/Transforms/LICM/call-hoisting.ll | 39 +++-
.../LICM/hoist-inaccessiblemem-call.ll | 207 ++++++++++++++++++
llvm/test/Transforms/LICM/pr50367.ll | 4 +-
llvm/test/Transforms/LICM/pr54495.ll | 2 +-
llvm/test/Transforms/LICM/pr59324.ll | 4 +-
llvm/test/Transforms/LICM/store-hoisting.ll | 8 +-
9 files changed, 336 insertions(+), 74 deletions(-)
create mode 100644 llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 733c35ec797cd..4e8a3f09c4524 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2302,15 +2302,15 @@ collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
// For a given store instruction or writeonly call instruction, this function
// checks that there are no read or writes that conflict with the memory
-// access in the instruction
+// access in the instruction. Instead of scanning all memory accesses in the
+// loop, walk the MemorySSA use-def graph starting from the loop header's
+// MemoryPhi. This visits only MemoryDefs and MemoryUses that are reachable
+// through loop-internal accesses, skipping uses that have already been
+// optimized to point outside the loop.
static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
AAResults *AA, Loop *CurLoop,
SinkAndHoistLICMFlags &Flags) {
assert(isa<CallInst>(*I) || isa<StoreInst>(*I));
- // If there are more accesses than the Promotion cap, then give up as we're
- // not walking a list that long.
- if (Flags.tooManyMemoryAccesses())
- return false;
auto *IMD = MSSA->getMemoryAccess(I);
BatchAAResults BAA(*AA);
@@ -2319,54 +2319,77 @@ static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
if (!MSSA->isLiveOnEntryDef(Source) && CurLoop->contains(Source->getBlock()))
return false;
- // If there are interfering Uses (i.e. their defining access is in the
- // loop), or ordered loads (stored as Defs!), don't move this store.
- // Could do better here, but this is conservatively correct.
- // TODO: Cache set of Uses on the first walk in runOnLoop, update when
- // moving accesses. Can also extend to dominating uses.
- for (auto *BB : CurLoop->getBlocks()) {
- auto *Accesses = MSSA->getBlockAccesses(BB);
- if (!Accesses)
- continue;
- for (const auto &MA : *Accesses)
- if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
- auto *MD = getClobberingMemoryAccess(*MSSA, BAA, Flags,
- const_cast<MemoryUse *>(MU));
- if (!MSSA->isLiveOnEntryDef(MD) && CurLoop->contains(MD->getBlock()))
- return false;
- // Disable hoisting past potentially interfering loads. Optimized
- // Uses may point to an access outside the loop, as getClobbering
- // checks the previous iteration when walking the backedge.
- // FIXME: More precise: no Uses that alias I.
- if (!Flags.getIsSink() && !MSSA->dominates(IMD, MU))
- return false;
- } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
+ // Walk the MemorySSA graph from the loop header's MemoryPhi. Every
+ // MemoryDef and (in optimized MSSA) every aliasing MemoryUse in the loop
+ // is reachable through the users of the MemoryPhi and loop-internal defs.
+ auto *HeaderPhi = MSSA->getMemoryAccess(CurLoop->getHeader());
+ if (!HeaderPhi)
+ return true;
+
+ SmallVector<const MemoryAccess *, 8> Worklist;
+ SmallPtrSet<const MemoryAccess *, 8> Visited;
+ Worklist.push_back(HeaderPhi);
+ Visited.insert(HeaderPhi);
+
+ while (!Worklist.empty()) {
+ const MemoryAccess *MA = Worklist.pop_back_val();
+ for (const User *U : MA->users()) {
+ const auto *UserMA = cast<MemoryAccess>(U);
+ if (!Visited.insert(UserMA).second)
+ continue;
+ // Skip accesses outside the loop (e.g., LCSSA phi users).
+ if (!CurLoop->contains(UserMA->getBlock()))
+ continue;
+
+ if (const auto *MU = dyn_cast<MemoryUse>(UserMA)) {
+ // If this use's defining access is IMD, it reads the location I
+ // writes. Since I is loop-invariant (its clobber is outside the
+ // loop, checked above), this read sees the same value every
+ // iteration and does not block hoisting.
+ if (MU->getDefiningAccess() == IMD)
+ continue;
+ // A MemoryUse whose defining access is inside the loop may or may
+ // not alias with I (MemorySSA uses may not be optimized yet).
+ // Check directly whether this use reads from I's write location.
+ Instruction *UseInst = MU->getMemoryInst();
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ if (isRefSet(BAA.getModRefInfo(UseInst, MemoryLocation::get(SI))))
+ return false;
+ } else {
+ if (UseInst != I &&
+ isRefSet(BAA.getModRefInfo(UseInst, cast<CallInst>(I))))
+ return false;
+ }
+ continue;
+ }
+
+ if (const auto *MD = dyn_cast<MemoryDef>(UserMA)) {
+ // Ordered loads are stored as MemoryDefs; always reject.
if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
- (void)LI; // Silence warning.
+ (void)LI;
assert(!LI->isUnordered() && "Expected unordered load");
return false;
}
- // Any call, while it may not be clobbering I, it may be a use.
+ // A call may read from the location written by I.
if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
- // Check if the call may read from the memory location written
- // to by I. Check CI's attributes and arguments; the number of
- // such checks performed is limited above by NoOfMemAccTooLarge.
- if (auto *SI = dyn_cast<StoreInst>(I)) {
- ModRefInfo MRI = BAA.getModRefInfo(CI, MemoryLocation::get(SI));
- if (isModOrRefSet(MRI))
- return false;
- } else {
- auto *SCI = cast<CallInst>(I);
- // If the instruction we are wanting to hoist is also a call
- // instruction then we need not check mod/ref info with itself
- if (SCI == CI)
- continue;
- ModRefInfo MRI = BAA.getModRefInfo(CI, SCI);
- if (isModOrRefSet(MRI))
- return false;
+ if (CI != I) {
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ if (isModOrRefSet(BAA.getModRefInfo(CI, MemoryLocation::get(SI))))
+ return false;
+ } else {
+ if (isModOrRefSet(BAA.getModRefInfo(CI, cast<CallInst>(I))))
+ return false;
+ }
}
}
+ // Follow defs inside the loop to find their users.
+ Worklist.push_back(MD);
}
+
+ // MemoryPhis in sub-loops: follow them.
+ if (isa<MemoryPhi>(UserMA))
+ Worklist.push_back(UserMA);
+ }
}
return true;
}
diff --git a/llvm/test/Analysis/MemorySSA/pr43427.ll b/llvm/test/Analysis/MemorySSA/pr43427.ll
index 254fb1104c590..f9338de8c6ee0 100644
--- a/llvm/test/Analysis/MemorySSA/pr43427.ll
+++ b/llvm/test/Analysis/MemorySSA/pr43427.ll
@@ -2,9 +2,13 @@
; CHECK-LABEL: @f(i1 %arg)
+; CHECK: entry:
+; CHECK: ; [[NO1:.*]] = MemoryDef(liveOnEntry)
+; CHECK-NEXT: store i16 undef, ptr %e, align 1
+
; CHECK: lbl1:
-; CHECK-NEXT: ; [[NO4:.*]] = MemoryPhi({entry,liveOnEntry},{lbl1.backedge,[[NO9:.*]]})
-; CHECK-NEXT: ; [[NO2:.*]] = MemoryDef([[NO4]])
+; CHECK-NEXT: ; [[NO4:.*]] = MemoryPhi({entry,[[NO1]]},{lbl1.backedge,[[NO2:.*]]})
+; CHECK-NEXT: ; [[NO2]] = MemoryDef([[NO4]])
; CHECK-NEXT: call void @g()
; CHECK-NEXT: br i1 %arg, label %for.end, label %if.else
@@ -12,24 +16,20 @@
; CHECK-NEXT: br i1 %arg, label %lbl3, label %lbl2
; CHECK: lbl2:
-; CHECK-NEXT: ; [[NO8:.*]] = MemoryPhi({lbl3,[[NO7:.*]]},{for.end,[[NO2]]})
; CHECK-NEXT: br label %lbl3
; CHECK: lbl3:
-; CHECK-NEXT: [[NO7]] = MemoryPhi({lbl2,[[NO8]]},{for.end,2})
+; CHECK-NEXT: br i1 %arg, label %lbl2, label %cleanup
; CHECK: cleanup:
; CHECK-NEXT: MemoryUse([[NO2]])
; CHECK-NEXT: %cleanup.dest = load i32, ptr undef, align 1
; CHECK: lbl1.backedge:
-; CHECK-NEXT: [[NO9]] = MemoryPhi({cleanup,[[NO7]]},{if.else,2})
; CHECK-NEXT: br label %lbl1
; CHECK: cleanup.cont:
-; CHECK-NEXT: ; [[NO6:.*]] = MemoryDef([[NO7]])
-; CHECK-NEXT: store i16 undef, ptr %e, align 1
-; CHECK-NEXT: 3 = MemoryDef([[NO6]])
+; CHECK-NEXT: ; [[NO3:.*]] = MemoryDef([[NO2]])
; CHECK-NEXT: call void @g()
define void @f(i1 %arg) {
diff --git a/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
index 0d32e508edf5f..07352a299f76b 100644
--- a/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
+++ b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
@@ -9,10 +9,12 @@ define void @f() {
; CHECK-LABEL: define void @f() {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[L_87_I:%.*]] = alloca [9 x i16], align 16
-; CHECK-NEXT: [[G_58_PROMOTED:%.*]] = load i32, ptr @g_58, align 4, !tbaa [[INT_TBAA0:![0-9]+]]
+; CHECK-NEXT: store ptr @g_58, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: [[TMP2_PROMOTED:%.*]] = load i32, ptr [[TMP2]], align 4, !tbaa [[INT_TBAA4:![0-9]+]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP31:%.*]] = phi i32 [ [[G_58_PROMOTED]], %[[ENTRY]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP31:%.*]] = phi i32 [ [[TMP2_PROMOTED]], %[[ENTRY]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[INC12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[OR]] = or i32 [[TMP31]], 10
; CHECK-NEXT: [[INC]] = add nsw i32 [[INC12]], 1
@@ -20,8 +22,7 @@ define void @f() {
; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[FOR_BODY]] ]
-; CHECK-NEXT: store ptr @g_58, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA4:![0-9]+]]
-; CHECK-NEXT: store i32 [[OR_LCSSA]], ptr @g_58, align 4, !tbaa [[INT_TBAA0]]
+; CHECK-NEXT: store i32 [[OR_LCSSA]], ptr [[TMP2]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: ret void
;
@@ -52,10 +53,10 @@ for.end: ; preds = %for.inc
!5 = !{!"any pointer", !1}
!6 = !{!"int", !1}
;.
-; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]]}
+; CHECK: [[ANYPTR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"any pointer", [[META2:![0-9]+]]}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]]}
; CHECK: [[META3]] = !{!"Simple C/C++ TBAA"}
-; CHECK: [[ANYPTR_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
-; CHECK: [[META5]] = !{!"any pointer", [[META2]]}
+; CHECK: [[INT_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"int", [[META2]]}
;.
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
index bb28d1ca93233..fdf37c70d24c9 100644
--- a/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -277,6 +277,35 @@ exit:
ret i32 %val
}
+define i32 @unrelated_read(ptr noalias %loc, ptr noalias %otherloc) {
+; CHECK-LABEL: define i32 @unrelated_read(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[OTHERLOC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VAL:%.*]] = call i32 @load(i32 [[IV]], ptr [[OTHERLOC]])
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi i32 [ [[VAL]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[VAL_LCSSA]]
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %val = call i32 @load(i32 %iv, ptr %otherloc)
+ call void @store(i32 0, ptr %loc)
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp slt i32 %iv, 200
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret i32 %val
+}
+
define void @neg_lv_value(ptr %loc) {
; CHECK-LABEL: define void @neg_lv_value(
; CHECK-SAME: ptr [[LOC:%.*]]) {
@@ -368,18 +397,18 @@ exit:
define void @neg_ref(ptr %loc) {
; CHECK-LABEL: define void @neg_ref(
; CHECK-SAME: ptr [[LOC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NEXT: [[LOOP:.*]]:
; CHECK-NEXT: call void @store(i32 0, ptr [[LOC]])
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[LOC]], align 4
; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 [[V]], 198
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
; CHECK-NEXT: br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
; CHECK: [[BACKEDGE]]:
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT2:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
; CHECK: [[EXIT2]]:
diff --git a/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll b/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
new file mode 100644
index 0000000000000..0f9bed4881db6
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<target-ir>,loop-mssa(licm)' < %s -S | FileCheck %s
+
+;; It should hoist fn_write_inaccessible_mem
+;; because there is no conflict between inaccessible memory and argmem
+define dso_local i32 @hoist_inaccessible_with_argmem(i32 %x, ptr %a, ptr %b) #0 {
+; CHECK-LABEL: define dso_local i32 @hoist_inaccessible_with_argmem(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[VAL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 1
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[GEP:%.*]], %[[LOOP]] ], [ [[VAL]], %[[ENTRY]] ]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[VAL2:%.*]] = call i32 @fn_args(ptr [[GEP2]])
+; CHECK-NEXT: [[GEP]] = getelementptr inbounds nuw i32, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[ACC:%.*]] = add nuw nsw i32 [[VAL2]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ACC]], 10
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[AFTER_LOOP:.*]]
+; CHECK: [[AFTER_LOOP]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[ACC_LCSSA]]
+;
+entry:
+ %val = getelementptr inbounds nuw i32, ptr %a, i64 1
+ br label %loop
+loop:
+ %phi = phi ptr [ %gep, %loop ], [ %val, %entry ]
+ %44 = load i32, ptr %phi, align 16
+ %gep2 = getelementptr i8, ptr %phi, i64 0
+ %val2 = call i32 @fn_args(ptr %gep2)
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ %gep = getelementptr inbounds nuw i32, ptr %phi, i64 0
+ %acc = add nuw nsw i32 %val2, 1
+ %cmp = icmp ult i32 %acc, 10
+ br i1 %cmp, label %loop, label %after_loop
+after_loop:
+ ret i32 %acc
+}
+
+declare i32 @fn_args(ptr) nounwind willreturn
+memory(argmem: read)
+
+;; It should NOT hoist fn_write_inaccessible_mem
+;; Because fn_args_2 reads inaccessible memory
+define dso_local i32 @no_hoist_inaccessible_conflict(i32 %x, ptr %a, ptr %b) #0 {
+; CHECK-LABEL: define dso_local i32 @no_hoist_inaccessible_conflict(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[VAL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[GEP:%.*]], %[[LOOP]] ], [ [[VAL]], %[[ENTRY]] ]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[VAL2:%.*]] = call i32 @fn_args_2(ptr [[GEP2]])
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: [[GEP]] = getelementptr inbounds nuw i32, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[ACC:%.*]] = add nuw nsw i32 [[VAL2]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ACC]], 10
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[AFTER_LOOP:.*]]
+; CHECK: [[AFTER_LOOP]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[ACC_LCSSA]]
+;
+entry:
+ %val = getelementptr inbounds nuw i32, ptr %a, i64 1
+ br label %loop
+loop:
+ %phi = phi ptr [ %gep, %loop ], [ %val, %entry ]
+ %44 = load i32, ptr %phi, align 16
+ %gep2 = getelementptr i8, ptr %phi, i64 0
+ %val2 = call i32 @fn_args_2(ptr %gep2)
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ %gep = getelementptr inbounds nuw i32, ptr %phi, i64 0
+ %acc = add nuw nsw i32 %val2, 1
+ %cmp = icmp ult i32 %acc, 10
+ br i1 %cmp, label %loop, label %after_loop
+after_loop:
+ ret i32 %acc
+}
+declare i32 @fn_args_2(ptr) nounwind willreturn
+memory(inaccessiblemem:read)
+
+;; Should hoist fn_write_inaccessible_mem
+;; It does not alias with store to noalias ptr
+define void @hoist_inaccessible_with_store(ptr noalias %loc, ptr noalias %loc2){
+; CHECK-LABEL: define void @hoist_inaccessible_with_store(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[LOC2]], align 4
+; CHECK-NEXT: store i32 [[VAL]], ptr [[LOC]], align 4
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*:]]
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: br label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+for.cond.cleanup:
+ ret void
+for.body:
+ %val = load i32, ptr %loc2
+ store i32 %val, ptr %loc
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %for.body
+}
+
+;; Should NOT hoist fn_write_inaccessible_mem
+;; because fn_read_inaccessible_mem reads the same memory location
+define void @no_hoist_inaccessible_read_conflict(ptr noalias %loc, ptr noalias %loc2){
+; CHECK-LABEL: define void @no_hoist_inaccessible_read_conflict(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[LOC2]], align 4
+; CHECK-NEXT: store i32 [[VAL]], ptr [[LOC]], align 4
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+for.body:
+ %val = load i32, ptr %loc2
+ store i32 %val, ptr %loc
+ call void @fn_read_inaccessible_mem()
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %for.body
+}
+
+;; Nothing should be hoisted from the loop because volatile
+define void @no_hoist_volatile(ptr %loc, ptr %loc2) {
+; CHECK-LABEL: define void @no_hoist_volatile(
+; CHECK-SAME: ptr [[LOC:%.*]], ptr [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: store volatile i32 0, ptr [[LOC]], align 4
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP]]
+;
+entry:
+ br label %loop
+loop:
+ %val = load i32, ptr %loc2
+ store volatile i32 0, ptr %loc
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %loop
+}
+
+;; Should hoist FPMR write (inaccessible mem) past unrelated vector load
+define void @hoist_inaccessible_past_load(ptr %p, ptr %q) #0 {
+; CHECK-LABEL: define void @hoist_inaccessible_past_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load <16 x i8>, ptr [[ADD_PTR]], align 1
+; CHECK-NEXT: call void @use_vec(<16 x i8> [[LD]])
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 1024
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %add.ptr = getelementptr i8, ptr %p, i32 %iv
+ %ld = load <16 x i8>, ptr %add.ptr, align 1
+ call void @fn_write_inaccessible_mem()
+ call void @use_vec(<16 x i8> %ld)
+ %iv.next = add i32 %iv, 16
+ %cmp = icmp slt i32 %iv, 1024
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret void
+}
+
+declare void @fn_write_inaccessible_mem()#0
+ memory(inaccessiblemem: write)
+
+declare void @fn_read_inaccessible_mem()#0
+ memory(inaccessiblemem: read)
+
+declare void @use_vec(<16 x i8>) #0
+ memory(argmem: read)
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind}
diff --git a/llvm/test/Transforms/LICM/pr50367.ll b/llvm/test/Transforms/LICM/pr50367.ll
index 6aafff74f61d8..3e3e5c92cd481 100644
--- a/llvm/test/Transforms/LICM/pr50367.ll
+++ b/llvm/test/Transforms/LICM/pr50367.ll
@@ -44,6 +44,9 @@ define void @store_null(i1 %arg) {
; CHECK-LABEL: define void @store_null(
; CHECK-SAME: i1 [[ARG:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: store ptr null, ptr @e, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: [[PTR:%.*]] = load ptr, ptr @e, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: store i32 0, ptr [[PTR]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[LOOP2:.*]]
@@ -53,7 +56,6 @@ define void @store_null(i1 %arg) {
; CHECK-NEXT: store i32 0, ptr null, align 4
; CHECK-NEXT: br label %[[LOOP2]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: store i32 0, ptr null, align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: br label %[[LOOP1]]
;
entry:
diff --git a/llvm/test/Transforms/LICM/pr54495.ll b/llvm/test/Transforms/LICM/pr54495.ll
index d01ca69d55242..5e66758257ef0 100644
--- a/llvm/test/Transforms/LICM/pr54495.ll
+++ b/llvm/test/Transforms/LICM/pr54495.ll
@@ -6,6 +6,7 @@
define void @test(ptr %p1, ptr %p2, ptr noalias %p3) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
+; CHECK-NEXT: store ptr [[P3:%.*]], ptr [[P3]], align 8
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P1:%.*]], [[ENTRY:%.*]] ], [ [[P2:%.*]], [[LOOP]] ]
@@ -13,7 +14,6 @@ define void @test(ptr %p1, ptr %p2, ptr noalias %p3) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[V]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[LOOP_EXIT:%.*]]
; CHECK: loop.exit:
-; CHECK-NEXT: store ptr [[P3:%.*]], ptr [[P3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LICM/pr59324.ll b/llvm/test/Transforms/LICM/pr59324.ll
index ec33a0f8ded0f..bb5570fec8b56 100644
--- a/llvm/test/Transforms/LICM/pr59324.ll
+++ b/llvm/test/Transforms/LICM/pr59324.ll
@@ -4,10 +4,10 @@
define void @test(ptr %a) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
; CHECK-NEXT: store ptr null, ptr null, align 8
; CHECK-NEXT: [[P:%.*]] = load ptr, ptr null, align 8
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT: store i32 [[V]], ptr [[A:%.*]], align 4
; CHECK-NEXT: br label [[LOOP]]
diff --git a/llvm/test/Transforms/LICM/store-hoisting.ll b/llvm/test/Transforms/LICM/store-hoisting.ll
index 016364fff9fbb..4ca7b4d48409a 100644
--- a/llvm/test/Transforms/LICM/store-hoisting.ll
+++ b/llvm/test/Transforms/LICM/store-hoisting.ll
@@ -188,20 +188,20 @@ define void @neg_ref(ptr %loc) {
; CHECK-LABEL: define void @neg_ref(
; CHECK-SAME: ptr [[LOC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[LOC]], align 4
+; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 [[V]], 198
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
-; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 0, 198
; CHECK-NEXT: br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
; CHECK: [[BACKEDGE]]:
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
; CHECK: [[EXIT1]]:
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: ret void
; CHECK: [[EXIT2]]:
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -588,10 +588,10 @@ define void @test_dominated_readonly(ptr %loc) {
; CHECK-LABEL: define void @test_dominated_readonly(
; CHECK-SAME: ptr [[LOC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: call void @readonly()
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
More information about the llvm-commits
mailing list