[llvm] [LICM] Rewrite noConflictingReadWrites to walk MemorySSA graph (PR #191468)
Sebastian Pop via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 12:54:11 PDT 2026
https://github.com/sebpop updated https://github.com/llvm/llvm-project/pull/191468
>From 13b6e394359cf5b7c6770dcc8c23af9137f1e6dd Mon Sep 17 00:00:00 2001
From: Sebastian Pop <spop at nvidia.com>
Date: Fri, 10 Apr 2026 10:53:24 -0500
Subject: [PATCH] [LICM] Rewrite noConflictingReadWrites to walk MemorySSA
graph
Instead of scanning all memory accesses in the loop and checking each
one with a clobber walk, traverse the MemorySSA use-def graph starting
from the loop header's MemoryPhi. In optimized MemorySSA, non-aliasing
MemoryUses have their defining access redirected outside the loop, so
they naturally won't appear as users of loop-internal accesses.
This removes: the O(all accesses) block scan, the tooManyMemoryAccesses
bailout, the incorrect dominance check (PR #187529), and the FIXME
about imprecise alias checking for MemoryUses.
---
llvm/lib/Transforms/Scalar/LICM.cpp | 113 ++++++----
llvm/test/Analysis/MemorySSA/pr43427.ll | 16 +-
.../2011-04-06-PromoteResultOfPromotion.ll | 17 +-
llvm/test/Transforms/LICM/call-hoisting.ll | 39 +++-
.../Transforms/LICM/hoist-aarch64-fpmr.ll | 69 ++++++
.../LICM/hoist-inaccessiblemem-call.ll | 207 ++++++++++++++++++
llvm/test/Transforms/LICM/pr50367.ll | 4 +-
llvm/test/Transforms/LICM/pr54495.ll | 2 +-
llvm/test/Transforms/LICM/pr59324.ll | 4 +-
llvm/test/Transforms/LICM/store-hoisting.ll | 8 +-
10 files changed, 405 insertions(+), 74 deletions(-)
create mode 100644 llvm/test/Transforms/LICM/hoist-aarch64-fpmr.ll
create mode 100644 llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 733c35ec797cd..4e8a3f09c4524 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -2302,15 +2302,15 @@ collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
// For a given store instruction or writeonly call instruction, this function
// checks that there are no read or writes that conflict with the memory
-// access in the instruction
+// access in the instruction. Instead of scanning all memory accesses in the
+// loop, walk the MemorySSA use-def graph starting from the loop header's
+// MemoryPhi. This visits only MemoryDefs and MemoryUses that are reachable
+// through loop-internal accesses, skipping uses that have already been
+// optimized to point outside the loop.
static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
AAResults *AA, Loop *CurLoop,
SinkAndHoistLICMFlags &Flags) {
assert(isa<CallInst>(*I) || isa<StoreInst>(*I));
- // If there are more accesses than the Promotion cap, then give up as we're
- // not walking a list that long.
- if (Flags.tooManyMemoryAccesses())
- return false;
auto *IMD = MSSA->getMemoryAccess(I);
BatchAAResults BAA(*AA);
@@ -2319,54 +2319,77 @@ static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
if (!MSSA->isLiveOnEntryDef(Source) && CurLoop->contains(Source->getBlock()))
return false;
- // If there are interfering Uses (i.e. their defining access is in the
- // loop), or ordered loads (stored as Defs!), don't move this store.
- // Could do better here, but this is conservatively correct.
- // TODO: Cache set of Uses on the first walk in runOnLoop, update when
- // moving accesses. Can also extend to dominating uses.
- for (auto *BB : CurLoop->getBlocks()) {
- auto *Accesses = MSSA->getBlockAccesses(BB);
- if (!Accesses)
- continue;
- for (const auto &MA : *Accesses)
- if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
- auto *MD = getClobberingMemoryAccess(*MSSA, BAA, Flags,
- const_cast<MemoryUse *>(MU));
- if (!MSSA->isLiveOnEntryDef(MD) && CurLoop->contains(MD->getBlock()))
- return false;
- // Disable hoisting past potentially interfering loads. Optimized
- // Uses may point to an access outside the loop, as getClobbering
- // checks the previous iteration when walking the backedge.
- // FIXME: More precise: no Uses that alias I.
- if (!Flags.getIsSink() && !MSSA->dominates(IMD, MU))
- return false;
- } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
+ // Walk the MemorySSA graph from the loop header's MemoryPhi. Every
+ // MemoryDef and (in optimized MSSA) every aliasing MemoryUse in the loop
+ // is reachable through the users of the MemoryPhi and loop-internal defs.
+ auto *HeaderPhi = MSSA->getMemoryAccess(CurLoop->getHeader());
+ if (!HeaderPhi)
+ return true;
+
+ SmallVector<const MemoryAccess *, 8> Worklist;
+ SmallPtrSet<const MemoryAccess *, 8> Visited;
+ Worklist.push_back(HeaderPhi);
+ Visited.insert(HeaderPhi);
+
+ while (!Worklist.empty()) {
+ const MemoryAccess *MA = Worklist.pop_back_val();
+ for (const User *U : MA->users()) {
+ const auto *UserMA = cast<MemoryAccess>(U);
+ if (!Visited.insert(UserMA).second)
+ continue;
+ // Skip accesses outside the loop (e.g., LCSSA phi users).
+ if (!CurLoop->contains(UserMA->getBlock()))
+ continue;
+
+ if (const auto *MU = dyn_cast<MemoryUse>(UserMA)) {
+ // If this use's defining access is IMD, it reads the location I
+ // writes. Since I is loop-invariant (its clobber is outside the
+ // loop, checked above), this read sees the same value every
+ // iteration and does not block hoisting.
+ if (MU->getDefiningAccess() == IMD)
+ continue;
+ // A MemoryUse whose defining access is inside the loop may or may
+ // not alias with I (MemorySSA uses may not be optimized yet).
+ // Check directly whether this use reads from I's write location.
+ Instruction *UseInst = MU->getMemoryInst();
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ if (isRefSet(BAA.getModRefInfo(UseInst, MemoryLocation::get(SI))))
+ return false;
+ } else {
+ if (UseInst != I &&
+ isRefSet(BAA.getModRefInfo(UseInst, cast<CallInst>(I))))
+ return false;
+ }
+ continue;
+ }
+
+ if (const auto *MD = dyn_cast<MemoryDef>(UserMA)) {
+ // Ordered loads are stored as MemoryDefs; always reject.
if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
- (void)LI; // Silence warning.
+ (void)LI;
assert(!LI->isUnordered() && "Expected unordered load");
return false;
}
- // Any call, while it may not be clobbering I, it may be a use.
+ // A call may read from the location written by I.
if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
- // Check if the call may read from the memory location written
- // to by I. Check CI's attributes and arguments; the number of
- // such checks performed is limited above by NoOfMemAccTooLarge.
- if (auto *SI = dyn_cast<StoreInst>(I)) {
- ModRefInfo MRI = BAA.getModRefInfo(CI, MemoryLocation::get(SI));
- if (isModOrRefSet(MRI))
- return false;
- } else {
- auto *SCI = cast<CallInst>(I);
- // If the instruction we are wanting to hoist is also a call
- // instruction then we need not check mod/ref info with itself
- if (SCI == CI)
- continue;
- ModRefInfo MRI = BAA.getModRefInfo(CI, SCI);
- if (isModOrRefSet(MRI))
- return false;
+ if (CI != I) {
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ if (isModOrRefSet(BAA.getModRefInfo(CI, MemoryLocation::get(SI))))
+ return false;
+ } else {
+ if (isModOrRefSet(BAA.getModRefInfo(CI, cast<CallInst>(I))))
+ return false;
+ }
}
}
+ // Follow defs inside the loop to find their users.
+ Worklist.push_back(MD);
}
+
+ // MemoryPhis in sub-loops: follow them.
+ if (isa<MemoryPhi>(UserMA))
+ Worklist.push_back(UserMA);
+ }
}
return true;
}
diff --git a/llvm/test/Analysis/MemorySSA/pr43427.ll b/llvm/test/Analysis/MemorySSA/pr43427.ll
index 254fb1104c590..f9338de8c6ee0 100644
--- a/llvm/test/Analysis/MemorySSA/pr43427.ll
+++ b/llvm/test/Analysis/MemorySSA/pr43427.ll
@@ -2,9 +2,13 @@
; CHECK-LABEL: @f(i1 %arg)
+; CHECK: entry:
+; CHECK: ; [[NO1:.*]] = MemoryDef(liveOnEntry)
+; CHECK-NEXT: store i16 undef, ptr %e, align 1
+
; CHECK: lbl1:
-; CHECK-NEXT: ; [[NO4:.*]] = MemoryPhi({entry,liveOnEntry},{lbl1.backedge,[[NO9:.*]]})
-; CHECK-NEXT: ; [[NO2:.*]] = MemoryDef([[NO4]])
+; CHECK-NEXT: ; [[NO4:.*]] = MemoryPhi({entry,[[NO1]]},{lbl1.backedge,[[NO2:.*]]})
+; CHECK-NEXT: ; [[NO2]] = MemoryDef([[NO4]])
; CHECK-NEXT: call void @g()
; CHECK-NEXT: br i1 %arg, label %for.end, label %if.else
@@ -12,24 +16,20 @@
; CHECK-NEXT: br i1 %arg, label %lbl3, label %lbl2
; CHECK: lbl2:
-; CHECK-NEXT: ; [[NO8:.*]] = MemoryPhi({lbl3,[[NO7:.*]]},{for.end,[[NO2]]})
; CHECK-NEXT: br label %lbl3
; CHECK: lbl3:
-; CHECK-NEXT: [[NO7]] = MemoryPhi({lbl2,[[NO8]]},{for.end,2})
+; CHECK-NEXT: br i1 %arg, label %lbl2, label %cleanup
; CHECK: cleanup:
; CHECK-NEXT: MemoryUse([[NO2]])
; CHECK-NEXT: %cleanup.dest = load i32, ptr undef, align 1
; CHECK: lbl1.backedge:
-; CHECK-NEXT: [[NO9]] = MemoryPhi({cleanup,[[NO7]]},{if.else,2})
; CHECK-NEXT: br label %lbl1
; CHECK: cleanup.cont:
-; CHECK-NEXT: ; [[NO6:.*]] = MemoryDef([[NO7]])
-; CHECK-NEXT: store i16 undef, ptr %e, align 1
-; CHECK-NEXT: 3 = MemoryDef([[NO6]])
+; CHECK-NEXT: ; [[NO3:.*]] = MemoryDef([[NO2]])
; CHECK-NEXT: call void @g()
define void @f(i1 %arg) {
diff --git a/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
index 0d32e508edf5f..07352a299f76b 100644
--- a/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
+++ b/llvm/test/Transforms/LICM/2011-04-06-PromoteResultOfPromotion.ll
@@ -9,10 +9,12 @@ define void @f() {
; CHECK-LABEL: define void @f() {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[L_87_I:%.*]] = alloca [9 x i16], align 16
-; CHECK-NEXT: [[G_58_PROMOTED:%.*]] = load i32, ptr @g_58, align 4, !tbaa [[INT_TBAA0:![0-9]+]]
+; CHECK-NEXT: store ptr @g_58, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: [[TMP2_PROMOTED:%.*]] = load i32, ptr [[TMP2]], align 4, !tbaa [[INT_TBAA4:![0-9]+]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP31:%.*]] = phi i32 [ [[G_58_PROMOTED]], %[[ENTRY]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP31:%.*]] = phi i32 [ [[TMP2_PROMOTED]], %[[ENTRY]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[INC12:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[OR]] = or i32 [[TMP31]], 10
; CHECK-NEXT: [[INC]] = add nsw i32 [[INC12]], 1
@@ -20,8 +22,7 @@ define void @f() {
; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY]], label %[[FOR_END:.*]]
; CHECK: [[FOR_END]]:
; CHECK-NEXT: [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[FOR_BODY]] ]
-; CHECK-NEXT: store ptr @g_58, ptr @g_116, align 8, !tbaa [[ANYPTR_TBAA4:![0-9]+]]
-; CHECK-NEXT: store i32 [[OR_LCSSA]], ptr @g_58, align 4, !tbaa [[INT_TBAA0]]
+; CHECK-NEXT: store i32 [[OR_LCSSA]], ptr [[TMP2]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: ret void
;
@@ -52,10 +53,10 @@ for.end: ; preds = %for.inc
!5 = !{!"any pointer", !1}
!6 = !{!"int", !1}
;.
-; CHECK: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]]}
+; CHECK: [[ANYPTR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"any pointer", [[META2:![0-9]+]]}
; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]]}
; CHECK: [[META3]] = !{!"Simple C/C++ TBAA"}
-; CHECK: [[ANYPTR_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
-; CHECK: [[META5]] = !{!"any pointer", [[META2]]}
+; CHECK: [[INT_TBAA4]] = !{[[META5:![0-9]+]], [[META5]], i64 0}
+; CHECK: [[META5]] = !{!"int", [[META2]]}
;.
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
index bb28d1ca93233..fdf37c70d24c9 100644
--- a/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -277,6 +277,35 @@ exit:
ret i32 %val
}
+define i32 @unrelated_read(ptr noalias %loc, ptr noalias %otherloc) {
+; CHECK-LABEL: define i32 @unrelated_read(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[OTHERLOC:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VAL:%.*]] = call i32 @load(i32 [[IV]], ptr [[OTHERLOC]])
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[VAL_LCSSA:%.*]] = phi i32 [ [[VAL]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[VAL_LCSSA]]
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %val = call i32 @load(i32 %iv, ptr %otherloc)
+ call void @store(i32 0, ptr %loc)
+ %iv.next = add i32 %iv, 1
+ %cmp = icmp slt i32 %iv, 200
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret i32 %val
+}
+
define void @neg_lv_value(ptr %loc) {
; CHECK-LABEL: define void @neg_lv_value(
; CHECK-SAME: ptr [[LOC:%.*]]) {
@@ -368,18 +397,18 @@ exit:
define void @neg_ref(ptr %loc) {
; CHECK-LABEL: define void @neg_ref(
; CHECK-SAME: ptr [[LOC:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
+; CHECK-NEXT: [[LOOP:.*]]:
; CHECK-NEXT: call void @store(i32 0, ptr [[LOC]])
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[LOC]], align 4
; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 [[V]], 198
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[LOOP]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
; CHECK-NEXT: br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
; CHECK: [[BACKEDGE]]:
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP1]], label %[[EXIT2:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
; CHECK: [[EXIT2]]:
diff --git a/llvm/test/Transforms/LICM/hoist-aarch64-fpmr.ll b/llvm/test/Transforms/LICM/hoist-aarch64-fpmr.ll
new file mode 100644
index 0000000000000..4813b12217d99
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-aarch64-fpmr.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes='loop-mssa(licm)' -S %s | FileCheck %s
+;
+; Test that LICM can hoist an FPMR write (inaccessible memory) past
+; unrelated vector loads in a loop.
+;
+; Reduced from:
+; float16x8_t test(const uint8_t *A, const uint8_t *B, int N) {
+; fpm_t fpm = __arm_fpm_init();
+; fpm = __arm_set_fpm_src1_format(fpm, __ARM_FPM_E5M2);
+; fpm = __arm_set_fpm_src2_format(fpm, __ARM_FPM_E5M2);
+; fpm = __arm_set_fpm_lscale2(fpm, 5);
+; float16x8_t acc = vdupq_n_f16(0.0f);
+; for (int i = 0; i < 1024; i += 16) {
+; mfloat8x16_t za = vreinterpretq_mf8_u8(vld1q_u8(A + i));
+; mfloat8x8_t zb = vreinterpret_mf8_u8(vld1_u8(B + i));
+; acc = vdotq_lane_f16_mf8_fpm(acc, za, zb, 0, fpm);
+; }
+; return acc;
+; }
+
+define <8 x half> @fpmr_hoist(ptr readonly %A, ptr readonly %B) {
+; CHECK-LABEL: define <8 x half> @fpmr_hoist(
+; CHECK-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: call void @llvm.aarch64.set.fpmr(i64 21474836480)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi <8 x half> [ zeroinitializer, %[[ENTRY]] ], [ [[DOT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[PA:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LDA:%.*]] = load <16 x i8>, ptr [[PA]], align 1
+; CHECK-NEXT: [[PB:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[LDB:%.*]] = load <8 x i8>, ptr [[PB]], align 1
+; CHECK-NEXT: [[WIDE:%.*]] = shufflevector <8 x i8> [[LDB]], <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[DOT]] = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> [[ACC]], <16 x i8> [[LDA]], <16 x i8> [[WIDE]], i32 0)
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[IV]], 1008
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[DOT_LCSSA:%.*]] = phi <8 x half> [ [[DOT]], %[[LOOP]] ]
+; CHECK-NEXT: ret <8 x half> [[DOT_LCSSA]]
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc = phi <8 x half> [ zeroinitializer, %entry ], [ %dot, %loop ]
+ %pA = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ldA = load <16 x i8>, ptr %pA, align 1
+ %pB = getelementptr inbounds i8, ptr %B, i64 %iv
+ %ldB = load <8 x i8>, ptr %pB, align 1
+ %wide = shufflevector <8 x i8> %ldB, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ call void @llvm.aarch64.set.fpmr(i64 21474836480)
+ %dot = call <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half> %acc, <16 x i8> %ldA, <16 x i8> %wide, i32 0)
+ %iv.next = add nuw nsw i64 %iv, 16
+ %cmp = icmp ult i64 %iv, 1008
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret <8 x half> %dot
+}
+
+declare void @llvm.aarch64.set.fpmr(i64) #0
+declare <8 x half> @llvm.aarch64.neon.fp8.fdot2.lane.v8f16.v16i8(<8 x half>, <16 x i8>, <16 x i8>, i32 immarg) #1
+
+attributes #0 = { mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #1 = { mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: read) }
diff --git a/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll b/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
new file mode 100644
index 0000000000000..0f9bed4881db6
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-inaccessiblemem-call.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<target-ir>,loop-mssa(licm)' < %s -S | FileCheck %s
+
+;; It should hoist fn_write_inaccessible_mem
+;; because there is no conflict between inaccessible memory and argmem
+define dso_local i32 @hoist_inaccessible_with_argmem(i32 %x, ptr %a, ptr %b) #0 {
+; CHECK-LABEL: define dso_local i32 @hoist_inaccessible_with_argmem(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[VAL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 1
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[GEP:%.*]], %[[LOOP]] ], [ [[VAL]], %[[ENTRY]] ]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[VAL2:%.*]] = call i32 @fn_args(ptr [[GEP2]])
+; CHECK-NEXT: [[GEP]] = getelementptr inbounds nuw i32, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[ACC:%.*]] = add nuw nsw i32 [[VAL2]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ACC]], 10
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[AFTER_LOOP:.*]]
+; CHECK: [[AFTER_LOOP]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[ACC_LCSSA]]
+;
+entry:
+ %val = getelementptr inbounds nuw i32, ptr %a, i64 1
+ br label %loop
+loop:
+ %phi = phi ptr [ %gep, %loop ], [ %val, %entry ]
+ %44 = load i32, ptr %phi, align 16
+ %gep2 = getelementptr i8, ptr %phi, i64 0
+ %val2 = call i32 @fn_args(ptr %gep2)
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ %gep = getelementptr inbounds nuw i32, ptr %phi, i64 0
+ %acc = add nuw nsw i32 %val2, 1
+ %cmp = icmp ult i32 %acc, 10
+ br i1 %cmp, label %loop, label %after_loop
+after_loop:
+ ret i32 %acc
+}
+
+declare i32 @fn_args(ptr) nounwind willreturn
+memory(argmem: read)
+
+;; It should NOT hoist fn_write_inaccessible_mem
+;; Because fn_args_2 reads inaccessible memory
+define dso_local i32 @no_hoist_inaccessible_conflict(i32 %x, ptr %a, ptr %b) #0 {
+; CHECK-LABEL: define dso_local i32 @no_hoist_inaccessible_conflict(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[VAL:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi ptr [ [[GEP:%.*]], %[[LOOP]] ], [ [[VAL]], %[[ENTRY]] ]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[VAL2:%.*]] = call i32 @fn_args_2(ptr [[GEP2]])
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: [[GEP]] = getelementptr inbounds nuw i32, ptr [[PHI]], i64 0
+; CHECK-NEXT: [[ACC:%.*]] = add nuw nsw i32 [[VAL2]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ACC]], 10
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[AFTER_LOOP:.*]]
+; CHECK: [[AFTER_LOOP]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC]], %[[LOOP]] ]
+; CHECK-NEXT: ret i32 [[ACC_LCSSA]]
+;
+entry:
+ %val = getelementptr inbounds nuw i32, ptr %a, i64 1
+ br label %loop
+loop:
+ %phi = phi ptr [ %gep, %loop ], [ %val, %entry ]
+ %44 = load i32, ptr %phi, align 16
+ %gep2 = getelementptr i8, ptr %phi, i64 0
+ %val2 = call i32 @fn_args_2(ptr %gep2)
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ %gep = getelementptr inbounds nuw i32, ptr %phi, i64 0
+ %acc = add nuw nsw i32 %val2, 1
+ %cmp = icmp ult i32 %acc, 10
+ br i1 %cmp, label %loop, label %after_loop
+after_loop:
+ ret i32 %acc
+}
+declare i32 @fn_args_2(ptr) nounwind willreturn
+memory(inaccessiblemem:read)
+
+;; Should hoist fn_write_inaccessible_mem
+;; It does not alias with store to noalias ptr
+define void @hoist_inaccessible_with_store(ptr noalias %loc, ptr noalias %loc2){
+; CHECK-LABEL: define void @hoist_inaccessible_with_store(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[LOC2]], align 4
+; CHECK-NEXT: store i32 [[VAL]], ptr [[LOC]], align 4
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_COND_CLEANUP:.*:]]
+; CHECK-NEXT: ret void
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: br label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+for.cond.cleanup:
+ ret void
+for.body:
+ %val = load i32, ptr %loc2
+ store i32 %val, ptr %loc
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %for.body
+}
+
+;; Should NOT hoist fn_write_inaccessible_mem
+;; because fn_read_inaccessible_mem reads the same memory location
+define void @no_hoist_inaccessible_read_conflict(ptr noalias %loc, ptr noalias %loc2){
+; CHECK-LABEL: define void @no_hoist_inaccessible_read_conflict(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[LOC2]], align 4
+; CHECK-NEXT: store i32 [[VAL]], ptr [[LOC]], align 4
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[FOR_BODY]]
+;
+entry:
+ br label %for.body
+for.body:
+ %val = load i32, ptr %loc2
+ store i32 %val, ptr %loc
+ call void @fn_read_inaccessible_mem()
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %for.body
+}
+
+;; Nothing should be hoisted from the loop because volatile
+define void @no_hoist_volatile(ptr %loc, ptr %loc2) {
+; CHECK-LABEL: define void @no_hoist_volatile(
+; CHECK-SAME: ptr [[LOC:%.*]], ptr [[LOC2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: store volatile i32 0, ptr [[LOC]], align 4
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: call void @fn_read_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP]]
+;
+entry:
+ br label %loop
+loop:
+ %val = load i32, ptr %loc2
+ store volatile i32 0, ptr %loc
+ call void @fn_write_inaccessible_mem()
+ call void @fn_read_inaccessible_mem()
+ br label %loop
+}
+
+;; Should hoist FPMR write (inaccessible mem) past unrelated vector load
+define void @hoist_inaccessible_past_load(ptr %p, ptr %q) #0 {
+; CHECK-LABEL: define void @hoist_inaccessible_past_load(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: call void @fn_write_inaccessible_mem()
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load <16 x i8>, ptr [[ADD_PTR]], align 1
+; CHECK-NEXT: call void @use_vec(<16 x i8> [[LD]])
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 16
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 1024
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+ %add.ptr = getelementptr i8, ptr %p, i32 %iv
+ %ld = load <16 x i8>, ptr %add.ptr, align 1
+ call void @fn_write_inaccessible_mem()
+ call void @use_vec(<16 x i8> %ld)
+ %iv.next = add i32 %iv, 16
+ %cmp = icmp slt i32 %iv, 1024
+ br i1 %cmp, label %loop, label %exit
+exit:
+ ret void
+}
+
+declare void @fn_write_inaccessible_mem()#0
+ memory(inaccessiblemem: write)
+
+declare void @fn_read_inaccessible_mem()#0
+ memory(inaccessiblemem: read)
+
+declare void @use_vec(<16 x i8>) #0
+ memory(argmem: read)
+
+attributes #0 = { mustprogress nofree norecurse nosync nounwind}
diff --git a/llvm/test/Transforms/LICM/pr50367.ll b/llvm/test/Transforms/LICM/pr50367.ll
index 6aafff74f61d8..3e3e5c92cd481 100644
--- a/llvm/test/Transforms/LICM/pr50367.ll
+++ b/llvm/test/Transforms/LICM/pr50367.ll
@@ -44,6 +44,9 @@ define void @store_null(i1 %arg) {
; CHECK-LABEL: define void @store_null(
; CHECK-SAME: i1 [[ARG:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: store ptr null, ptr @e, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: [[PTR:%.*]] = load ptr, ptr @e, align 8, !tbaa [[ANYPTR_TBAA0]]
+; CHECK-NEXT: store i32 0, ptr [[PTR]], align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
; CHECK-NEXT: br label %[[LOOP2:.*]]
@@ -53,7 +56,6 @@ define void @store_null(i1 %arg) {
; CHECK-NEXT: store i32 0, ptr null, align 4
; CHECK-NEXT: br label %[[LOOP2]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: store i32 0, ptr null, align 4, !tbaa [[INT_TBAA4]]
; CHECK-NEXT: br label %[[LOOP1]]
;
entry:
diff --git a/llvm/test/Transforms/LICM/pr54495.ll b/llvm/test/Transforms/LICM/pr54495.ll
index d01ca69d55242..5e66758257ef0 100644
--- a/llvm/test/Transforms/LICM/pr54495.ll
+++ b/llvm/test/Transforms/LICM/pr54495.ll
@@ -6,6 +6,7 @@
define void @test(ptr %p1, ptr %p2, ptr noalias %p3) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
+; CHECK-NEXT: store ptr [[P3:%.*]], ptr [[P3]], align 8
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P1:%.*]], [[ENTRY:%.*]] ], [ [[P2:%.*]], [[LOOP]] ]
@@ -13,7 +14,6 @@ define void @test(ptr %p1, ptr %p2, ptr noalias %p3) {
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[V]], 0
; CHECK-NEXT: br i1 [[CMP]], label [[LOOP]], label [[LOOP_EXIT:%.*]]
; CHECK: loop.exit:
-; CHECK-NEXT: store ptr [[P3:%.*]], ptr [[P3]], align 8
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LICM/pr59324.ll b/llvm/test/Transforms/LICM/pr59324.ll
index ec33a0f8ded0f..bb5570fec8b56 100644
--- a/llvm/test/Transforms/LICM/pr59324.ll
+++ b/llvm/test/Transforms/LICM/pr59324.ll
@@ -4,10 +4,10 @@
define void @test(ptr %a) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
; CHECK-NEXT: store ptr null, ptr null, align 8
; CHECK-NEXT: [[P:%.*]] = load ptr, ptr null, align 8
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4
; CHECK-NEXT: store i32 [[V]], ptr [[A:%.*]], align 4
; CHECK-NEXT: br label [[LOOP]]
diff --git a/llvm/test/Transforms/LICM/store-hoisting.ll b/llvm/test/Transforms/LICM/store-hoisting.ll
index 016364fff9fbb..4ca7b4d48409a 100644
--- a/llvm/test/Transforms/LICM/store-hoisting.ll
+++ b/llvm/test/Transforms/LICM/store-hoisting.ll
@@ -188,20 +188,20 @@ define void @neg_ref(ptr %loc) {
; CHECK-LABEL: define void @neg_ref(
; CHECK-SAME: ptr [[LOC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
+; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[LOC]], align 4
+; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 [[V]], 198
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
-; CHECK-NEXT: [[EARLYCND:%.*]] = icmp eq i32 0, 198
; CHECK-NEXT: br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
; CHECK: [[BACKEDGE]]:
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT2:.*]]
; CHECK: [[EXIT1]]:
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: ret void
; CHECK: [[EXIT2]]:
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -588,10 +588,10 @@ define void @test_dominated_readonly(ptr %loc) {
; CHECK-LABEL: define void @test_dominated_readonly(
; CHECK-SAME: ptr [[LOC:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: store i32 0, ptr [[LOC]], align 4
; CHECK-NEXT: call void @readonly()
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200
More information about the llvm-commits
mailing list