[llvm] [PromoteMemToReg] Insert store undef when removing lifetime markers (PR #191909)
Alexandre Isoard via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 15:57:31 PDT 2026
https://github.com/isoard-amd updated https://github.com/llvm/llvm-project/pull/191909
>From 1b68d36890927978081f05aa1cc934995892e1af Mon Sep 17 00:00:00 2001
From: Alexandre Isoard <alexandre.isoard at amd.com>
Date: Mon, 13 Apr 2026 15:15:54 -0600
Subject: [PATCH 1/3] [PromoteMemToReg] Insert store undef when removing
lifetime markers
When an alloca is reused across loop iterations and has lifetime.start
at the top of the loop, SROA splits it into per-field slice allocas and
re-emits the lifetime markers on each slice. PromoteMemToReg then
promotes the slices but simply deletes the lifetime intrinsics, leaving
no reaching definition on the back-edge. The SSA construction must then
insert a recurrent PHI at the loop header to carry the value from the
previous iteration, even though the value is logically undef at the
start of each iteration.
Fix this by inserting a `store undef` to the alloca in place of each
lifetime.start or lifetime.end intrinsic before erasing it. This gives
mem2reg a concrete reaching definition at the lifetime boundary, so
back-edge PHIs whose only purpose is to propagate a stale value through
an undef point are eliminated. The value is instead exposed as undef
at the use site (typically a PHI at the first block where the alloca
is conditionally written).
Assisted-by: Claude
---
.../Utils/PromoteMemoryToRegister.cpp | 16 +-
.../Transforms/SROA/lifetime-aware-phi.ll | 192 ++++++++++++++++++
2 files changed, 205 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Transforms/SROA/lifetime-aware-phi.ll
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index ed0e864fd6905..131863bc2f83b 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -535,9 +535,8 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
}
if (!I->getType()->isVoidTy()) {
- // The only users of this bitcast/GEP instruction are lifetime intrinsics.
- // Follow the use/def chain to erase them now instead of leaving it for
- // dead code elimination later.
+ // Follow the use/def chain to erase users of this instruction now
+ // instead of leaving it for dead code elimination later.
for (Use &UU : llvm::make_early_inc_range(I->uses())) {
Instruction *Inst = cast<Instruction>(UU.getUser());
@@ -546,9 +545,20 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
Inst->dropDroppableUse(UU);
continue;
}
+
Inst->eraseFromParent();
}
}
+
+ // Same as above for lifetime intrinsics directly on the alloca.
+ if (auto *II = dyn_cast<IntrinsicInst>(I))
+ if (II->isLifetimeStartOrEnd()) {
+ auto *Store = new StoreInst(UndefValue::get(AI->getAllocatedType()), AI,
+ /*isVolatile=*/false, AI->getAlign(),
+ I->getIterator());
+ Store->setDebugLoc(II->getDebugLoc());
+ }
+
I->eraseFromParent();
}
}
diff --git a/llvm/test/Transforms/SROA/lifetime-aware-phi.ll b/llvm/test/Transforms/SROA/lifetime-aware-phi.ll
new file mode 100644
index 0000000000000..d0c902909cec5
--- /dev/null
+++ b/llvm/test/Transforms/SROA/lifetime-aware-phi.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=sroa -S | FileCheck %s
+
+; SROA should use lifetime intrinsics to avoid creating recurrent PHI nodes
+; when promoting struct allocas.
+;
+; The alloca is reused across loop iterations. Without lifetime markers, SROA
+; must conservatively assume the i32 field retains its value on the skip path,
+; so it inserts a back-edge PHI at the loop header carrying the previous
+; iteration's value.
+;
+; With lifetime.start at the top of the loop, the alloca's content is
+; logically undefined at that point each iteration. SROA should propagate this
+; onto each slice alloca: PromoteMemToReg can then treat lifetime.start as an
+; implicit "store undef", breaking the back-edge dependence and eliminating
+; the recurrent PHI.
+;
+; PromoteMemToReg treats lifetime markers as implicit "store undef", breaking
+; back-edge dependences and eliminating recurrent PHIs in @with_lifetime and
+; @with_lifetime_end.
+
+%struct.S = type { i8, i32, i8 }
+
+declare void @use(i32)
+declare void @llvm.lifetime.start.p0(ptr captures(none))
+declare void @llvm.lifetime.end.p0(ptr captures(none))
+
+; Without lifetime markers a recurrent PHI at the loop header is unavoidable.
+define void @without_lifetime(i32 %val, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @without_lifetime(
+; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[S_SROA_1_0:%.*]] = phi i32 [ undef, %[[ENTRY]] ], [ [[S_SROA_1_1:%.*]], %[[CLEANUP:.*]] ]
+; CHECK-NEXT: br i1 [[C1]], label %[[INIT:.*]], label %[[SKIP:.*]]
+; CHECK: [[INIT]]:
+; CHECK-NEXT: br label %[[READ:.*]]
+; CHECK: [[SKIP]]:
+; CHECK-NEXT: br label %[[READ]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[S_SROA_1_1]] = phi i32 [ [[VAL]], %[[INIT]] ], [ [[S_SROA_1_0]], %[[SKIP]] ]
+; CHECK-NEXT: call void @use(i32 [[S_SROA_1_1]])
+; CHECK-NEXT: br label %[[CLEANUP]]
+; CHECK: [[CLEANUP]]:
+; CHECK-NEXT: br i1 [[C2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %s = alloca %struct.S, align 4
+ br label %loop
+
+loop:
+ br i1 %c1, label %init, label %skip
+
+init:
+ %gep0 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 0
+ store i8 65, ptr %gep0, align 4
+ %gep1 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ store i32 %val, ptr %gep1, align 4
+ %gep2 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 2
+ store i8 90, ptr %gep2, align 4
+ br label %read
+
+skip:
+ br label %read
+
+read:
+ %gep1r = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ %v = load i32, ptr %gep1r, align 4
+ call void @use(i32 %v)
+ br label %cleanup
+
+cleanup:
+ br i1 %c2, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; With lifetime markers the back-edge PHI is unnecessary: lifetime.start
+; at the top of the loop makes the alloca's content undef at that point,
+; so the skip path should yield undef rather than carrying a stale value.
+define void @with_lifetime(i32 %val, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @with_lifetime(
+; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[C1]], label %[[INIT:.*]], label %[[SKIP:.*]]
+; CHECK: [[INIT]]:
+; CHECK-NEXT: br label %[[READ:.*]]
+; CHECK: [[SKIP]]:
+; CHECK-NEXT: br label %[[READ]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[S_SROA_31_0:%.*]] = phi i32 [ [[VAL]], %[[INIT]] ], [ undef, %[[SKIP]] ]
+; CHECK-NEXT: call void @use(i32 [[S_SROA_31_0]])
+; CHECK-NEXT: br label %[[CLEANUP:.*]]
+; CHECK: [[CLEANUP]]:
+; CHECK-NEXT: br i1 [[C2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; No recurrent PHI should be created at the loop header.
+; The only PHI is at the read block, merging %val (init) with undef (skip).
+entry:
+ %s = alloca %struct.S, align 4
+ br label %loop
+
+loop:
+ call void @llvm.lifetime.start.p0(ptr %s)
+ br i1 %c1, label %init, label %skip
+
+init:
+ %gep0 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 0
+ store i8 65, ptr %gep0, align 4
+ %gep1 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ store i32 %val, ptr %gep1, align 4
+ %gep2 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 2
+ store i8 90, ptr %gep2, align 4
+ br label %read
+
+skip:
+ br label %read
+
+read:
+ %gep1r = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ %v = load i32, ptr %gep1r, align 4
+ call void @use(i32 %v)
+ br label %cleanup
+
+cleanup:
+ call void @llvm.lifetime.end.p0(ptr %s)
+ br i1 %c2, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; lifetime.end shortens live ranges: after lifetime.end the value is undef,
+; so a PHI at the loop header carrying the value through the back-edge from
+; the block containing lifetime.end is unnecessary.
+define void @with_lifetime_end(i32 %val, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @with_lifetime_end(
+; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: br i1 [[C1]], label %[[INIT:.*]], label %[[SKIP:.*]]
+; CHECK: [[INIT]]:
+; CHECK-NEXT: br label %[[READ:.*]]
+; CHECK: [[SKIP]]:
+; CHECK-NEXT: br label %[[READ]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[S_SROA_31_0:%.*]] = phi i32 [ [[VAL]], %[[INIT]] ], [ undef, %[[SKIP]] ]
+; CHECK-NEXT: call void @use(i32 [[S_SROA_31_0]])
+; CHECK-NEXT: br i1 [[C2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+; No recurrent PHI at the loop header.
+entry:
+ %s = alloca %struct.S, align 4
+ br label %loop
+
+loop:
+ call void @llvm.lifetime.start.p0(ptr %s)
+ br i1 %c1, label %init, label %skip
+
+init:
+ %gep0 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 0
+ store i8 65, ptr %gep0, align 4
+ %gep1 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ store i32 %val, ptr %gep1, align 4
+ %gep2 = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 2
+ store i8 90, ptr %gep2, align 4
+ br label %read
+
+skip:
+ br label %read
+
+read:
+ %gep1r = getelementptr inbounds %struct.S, ptr %s, i32 0, i32 1
+ %v = load i32, ptr %gep1r, align 4
+ call void @use(i32 %v)
+ call void @llvm.lifetime.end.p0(ptr %s)
+ br i1 %c2, label %loop, label %exit
+
+exit:
+ ret void
+}
>From 7303a74da48052e4c230f146619fdfd5dc448016 Mon Sep 17 00:00:00 2001
From: Alexandre Isoard <alexandre.isoard at amd.com>
Date: Mon, 13 Apr 2026 16:41:03 -0600
Subject: [PATCH 2/3] [PromoteMemToReg] Update tests for
store-undef-at-lifetime-boundary change
The store undef inserted at lifetime.start/end boundaries now surfaces
in debug info (correct #dbg_value(undef) records reflecting that the
variable is undefined at that point) and affects value propagation
in phase-ordering tests. Update affected tests accordingly:
- assignment-tracking/sroa/{rewrite,memcpy}.ll: add CHECK lines for the
new #dbg_value(undef) records emitted at lifetime.start and
lifetime.end for promoted scalar slices.
- assignment-tracking/sroa/frag.ll: add CHECK-NEXT lines for the undef
records emitted for all promoted slices at the lifetime boundary.
- assignment-tracking/sroa/after-inlining.ll: the promoted scalar slice
is now undef at the lifetime.start; update the check from
#dbg_assign(ptr poison) to #dbg_value(undef), which still carries the
InlinedAt location the test was written to verify.
- Transforms/PhaseOrdering/always-inline-alloca-promotion.ll: regenerate
with update_test_checks.py; the alloca is now correctly seen as undef
across lifetime.start boundaries rather than carrying a stale value.
- The store undef inserted at lifetime boundaries changes phi node
ordering in loop headers. Regenerate with update_test_checks.py.
---
.../assignment-tracking/sroa/after-inlining.ll | 2 +-
.../Generic/assignment-tracking/sroa/frag.ll | 6 ++++--
.../Generic/assignment-tracking/sroa/memcpy.ll | 6 ++++++
.../Generic/assignment-tracking/sroa/rewrite.ll | 6 ++++++
.../Transforms/PhaseOrdering/ARM/arm_mean_q7.ll | 2 +-
.../Transforms/PhaseOrdering/ARM/arm_var_q31.ll | 14 +++++++-------
.../always-inline-alloca-promotion.ll | 6 ++----
7 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/after-inlining.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/after-inlining.ll
index ff4fffd37620a..2854a585c7e27 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/after-inlining.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/after-inlining.ll
@@ -28,7 +28,7 @@
;;
;; $ clang test.c -Xclang -fexperimental-assignment-tracking -O2 -g
-; CHECK: #dbg_assign(i1 false, !{{.+}}, !DIExpression(), !{{.+}}, ptr poison, !DIExpression(), ![[DBG:[0-9]+]]
+; CHECK: #dbg_value(i32 undef, !{{.+}}, !DIExpression(), ![[DBG:[0-9]+]]
; CHECK-DAG: ![[DBG]] = !DILocation(line: 0, scope: ![[INL_SC:[0-9]+]], inlinedAt: ![[IA:[0-9]+]])
; CHECK-DAG: ![[IA]] = distinct !DILocation(line: 21, column: 12, scope: ![[SC:[0-9]+]])
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/frag.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/frag.ll
index be0fa786aea71..1a186f0876896 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/frag.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/frag.ll
@@ -21,8 +21,10 @@
; CHECK: %call = call
; CHECK-NEXT: %0 = extractvalue { <2 x float>, <2 x float> } %call, 0
; CHECK-NEXT: %1 = extractvalue { <2 x float>, <2 x float> } %call, 1
-; CHECK-NEXT: #dbg_value(<2 x float> %0, ![[var:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 256, 64),
-; CHECK-NEXT: #dbg_value(<2 x float> %1, ![[var]], !DIExpression(DW_OP_LLVM_fragment, 320, 64),
+;; lifetime.end makes all promoted slices undef.
+; CHECK-NEXT: #dbg_value([2 x %class.c] undef, ![[var:[0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 0, 256),
+; CHECK-NEXT: #dbg_value(<2 x float> undef, ![[var]], !DIExpression(DW_OP_LLVM_fragment, 256, 64),
+; CHECK-NEXT: #dbg_value(<2 x float> undef, ![[var]], !DIExpression(DW_OP_LLVM_fragment, 320, 64),
%class.c = type { [4 x float] }
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
index ee7e9341ca59a..dde24533a94e8 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/memcpy.ll
@@ -31,6 +31,9 @@
; CHECK-NEXT: %To.sroa.4 = alloca { i32, i32, i32 }, align 8, !DIAssignID ![[ID_3:[0-9]+]]
; CHECK-NEXT: #dbg_assign({{.+}} undef, ![[TO]], !DIExpression(DW_OP_LLVM_fragment, 128, 96), ![[ID_3]], ptr %To.sroa.4, !DIExpression(),
+;; lifetime.start makes the middle (promoted) slice undef.
+; CHECK: #dbg_value(i32 undef, ![[TO]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
+
;; Split memcpy.
; CHECK: call void @llvm.memcpy{{.*}}(ptr align 8 %To.sroa.0, ptr align 4 @From, i64 12, i1 false),{{.*}}!DIAssignID ![[ID_4:[0-9]+]]
;; This slice has been split and is promoted.
@@ -42,6 +45,9 @@
; CHECK-NEXT: #dbg_value(i32 %To.sroa.3.0.copyload, ![[TO]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
; CHECK-NEXT: #dbg_assign({{.+}} undef, ![[TO]], !DIExpression(DW_OP_LLVM_fragment, 128, 96), ![[ID_6]], ptr %To.sroa.4, !DIExpression(),
+;; lifetime.end also makes the middle slice undef.
+; CHECK: #dbg_value(i32 undef, ![[TO]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
%struct.LargeStruct = type { i32, i32, i32, i32, i32, i32, i32 }
diff --git a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
index 73e954a27375f..bc60167c5b391 100644
--- a/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
+++ b/llvm/test/DebugInfo/Generic/assignment-tracking/sroa/rewrite.ll
@@ -31,6 +31,9 @@
; CHECK-NEXT: %S.sroa.5 = alloca { i32, i32, i32 }, align 8, !DIAssignID ![[ID_3:[0-9]+]]
; CHECK-NEXT: #dbg_assign(i1 undef, ![[VAR]], !DIExpression(DW_OP_LLVM_fragment, 128, 96), ![[ID_3]], ptr %S.sroa.5, !DIExpression(),
+;; lifetime.start makes the middle (promoted) slice undef.
+; CHECK: #dbg_value(i32 undef, ![[VAR]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
+
;; The memset has been sliced up (middle slice removed).
; CHECK: call void @llvm.memset{{.*}}(ptr align 8 %S.sroa.0, i8 0, i64 12, i1 false), !dbg !{{.+}}, !DIAssignID ![[ID_5:[0-9]+]]
; CHECK: call void @llvm.memset{{.*}}(ptr align 8 %S.sroa.5, i8 0, i64 12, i1 false), !dbg !{{.+}}, !DIAssignID ![[ID_6:[0-9]+]]
@@ -44,6 +47,9 @@
; CHECK-NEXT: %0 = load i32, ptr @Glob, align 4, !dbg !{{.+}}
; CHECK-NEXT: #dbg_value(i32 %0, ![[VAR]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
+;; lifetime.end also makes the middle slice undef.
+; CHECK: #dbg_value(i32 undef, ![[VAR]], !DIExpression(DW_OP_LLVM_fragment, 96, 32),
+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
%struct.LargeStruct = type { i32, i32, i32, i32, i32, i32, i32 }
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
index 9ff9f92c4edca..f7b60684ac963 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
@@ -14,8 +14,8 @@ define void @arm_mean_q7(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
; CHECK-NEXT: br i1 [[CMP_NOT10]], label [[WHILE_END:%.*]], label [[WHILE_BODY:%.*]]
; CHECK: while.body:
; CHECK-NEXT: [[SUM_013:%.*]] = phi i32 [ [[TMP2:%.*]], [[WHILE_BODY]] ], [ 0, [[WHILE_BODY_PREHEADER:%.*]] ]
-; CHECK-NEXT: [[PSRC_ADDR_012:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRC:%.*]], [[WHILE_BODY_PREHEADER]] ]
; CHECK-NEXT: [[BLKCNT_011:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[SHR]], [[WHILE_BODY_PREHEADER]] ]
+; CHECK-NEXT: [[PSRC_ADDR_012:%.*]] = phi ptr [ [[ADD_PTR:%.*]], [[WHILE_BODY]] ], [ [[PSRC:%.*]], [[WHILE_BODY_PREHEADER]] ]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[PSRC_ADDR_012]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = tail call i32 @llvm.arm.mve.addv.v16i8(<16 x i8> [[TMP0]], i32 0)
; CHECK-NEXT: [[TMP2]] = add i32 [[TMP1]], [[SUM_013]]
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_var_q31.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_var_q31.ll
index 2f89eb44fee4c..20daa8c86e4fb 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_var_q31.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_var_q31.ll
@@ -22,15 +22,15 @@ define void @arm_var_q31(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[WHILE_BODY_PREHEADER67:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[SHR]], 1073741820
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[SHR]], 3
; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[N_VEC]], 4
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PSRC:%.*]], i32 [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[SHR]], 3
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PSRC]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI53:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PSRC]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> <i32 0, i32 16, i32 32, i32 48>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 4
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = tail call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[VECTOR_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison), !tbaa [[TBAA4:![0-9]+]]
@@ -76,15 +76,15 @@ define void @arm_var_q31(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
; CHECK-NEXT: br i1 [[CMP_N]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY_PREHEADER67]]
; CHECK: while.body.preheader65:
; CHECK-NEXT: [[SUMOFSQUARES_043_PH:%.*]] = phi i64 [ 0, [[WHILE_BODY_PREHEADER]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[PSRC_ADDR_042_PH:%.*]] = phi ptr [ [[PSRC]], [[WHILE_BODY_PREHEADER]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT: [[BLKCNT_041_PH:%.*]] = phi i32 [ [[SHR]], [[WHILE_BODY_PREHEADER]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[SUM_040_PH:%.*]] = phi i64 [ 0, [[WHILE_BODY_PREHEADER]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[BLKCNT_041_PH:%.*]] = phi i32 [ [[SHR]], [[WHILE_BODY_PREHEADER]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[PSRC_ADDR_040_PH:%.*]] = phi ptr [ [[PSRC]], [[WHILE_BODY_PREHEADER]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
; CHECK-NEXT: br label [[WHILE_BODY:%.*]]
; CHECK: while.body:
; CHECK-NEXT: [[SUMOFSQUARES_043:%.*]] = phi i64 [ [[ADD27:%.*]], [[WHILE_BODY]] ], [ [[SUMOFSQUARES_043_PH]], [[WHILE_BODY_PREHEADER67]] ]
-; CHECK-NEXT: [[PSRC_ADDR_042:%.*]] = phi ptr [ [[INCDEC_PTR22:%.*]], [[WHILE_BODY]] ], [ [[PSRC_ADDR_042_PH]], [[WHILE_BODY_PREHEADER67]] ]
-; CHECK-NEXT: [[BLKCNT_041:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_041_PH]], [[WHILE_BODY_PREHEADER67]] ]
; CHECK-NEXT: [[SUM_040:%.*]] = phi i64 [ [[ADD29:%.*]], [[WHILE_BODY]] ], [ [[SUM_040_PH]], [[WHILE_BODY_PREHEADER67]] ]
+; CHECK-NEXT: [[BLKCNT_041:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[BLKCNT_041_PH]], [[WHILE_BODY_PREHEADER67]] ]
+; CHECK-NEXT: [[PSRC_ADDR_042:%.*]] = phi ptr [ [[INCDEC_PTR22:%.*]], [[WHILE_BODY]] ], [ [[PSRC_ADDR_040_PH]], [[WHILE_BODY_PREHEADER67]] ]
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[PSRC_ADDR_042]], i32 4
; CHECK-NEXT: [[TMP35:%.*]] = load i32, ptr [[PSRC_ADDR_042]], align 4, !tbaa [[TBAA4]]
; CHECK-NEXT: [[SHR2:%.*]] = ashr i32 [[TMP35]], 8
@@ -123,8 +123,8 @@ define void @arm_var_q31(ptr noundef %pSrc, i32 noundef %blockSize, ptr noundef
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[TMP39]]
; CHECK-NEXT: br label [[WHILE_END]]
; CHECK: while.end:
-; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i64 [ 0, [[IF_END]] ], [ [[ADD29_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
; CHECK-NEXT: [[PSRC_ADDR_0_LCSSA:%.*]] = phi ptr [ [[PSRC]], [[IF_END]] ], [ [[SCEVGEP]], [[WHILE_END_LOOPEXIT]] ]
+; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i64 [ 0, [[IF_END]] ], [ [[ADD29_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
; CHECK-NEXT: [[SUMOFSQUARES_0_LCSSA:%.*]] = phi i64 [ 0, [[IF_END]] ], [ [[ADD27_LCSSA]], [[WHILE_END_LOOPEXIT]] ]
; CHECK-NEXT: [[REM:%.*]] = and i32 [[BLOCKSIZE]], 3
; CHECK-NEXT: [[CMP31_NOT46:%.*]] = icmp eq i32 [[REM]], 0
diff --git a/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
index b235987c19d6b..519cfaed87ab4 100644
--- a/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
+++ b/llvm/test/Transforms/PhaseOrdering/always-inline-alloca-promotion.ll
@@ -12,12 +12,10 @@ define void @pluto() #0 {
; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr inttoptr (i64 48 to ptr), align 16
; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt i64 [[TMP1]], 0
-; CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 16 x float> @llvm.vector.insert.nxv16f32.nxv4f32(<vscale x 16 x float> zeroinitializer, <vscale x 4 x float> zeroinitializer, i64 0)
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> undef, i64 0)
+; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP2]], <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP3]]
; CHECK-NEXT: br label %[[SNORK_EXIT:.*]]
; CHECK: [[SNORK_EXIT]]:
-; CHECK-NEXT: [[DOT0:%.*]] = phi <vscale x 16 x float> [ undef, [[TMP0:%.*]] ], [ [[SPEC_SELECT:%.*]], %[[SNORK_EXIT]] ]
-; CHECK-NEXT: [[SPEC_SELECT]] = select i1 [[TMP2]], <vscale x 16 x float> [[TMP3]], <vscale x 16 x float> [[DOT0]]
-; CHECK-NEXT: [[TMP4:%.*]] = tail call <vscale x 4 x float> @llvm.vector.extract.nxv4f32.nxv16f32(<vscale x 16 x float> [[SPEC_SELECT]], i64 0)
; CHECK-NEXT: tail call void @llvm.aarch64.sme.mopa.nxv4f32(i32 0, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer, <vscale x 4 x float> zeroinitializer, <vscale x 4 x float> [[TMP4]])
; CHECK-NEXT: br label %[[SNORK_EXIT]]
;
>From 19297b9e9ac136bdb439db05e263056c5d6d5c72 Mon Sep 17 00:00:00 2001
From: Alexandre Isoard <alexandre.isoard at amd.com>
Date: Tue, 21 Apr 2026 17:06:35 -0600
Subject: [PATCH 3/3] fixup! [PromoteMemToReg] Insert store undef when removing
lifetime markers
---
.../Utils/PromoteMemoryToRegister.cpp | 35 ++++++++++++++-----
1 file changed, 27 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 131863bc2f83b..21ddc78f1469b 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -523,6 +523,22 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
// Knowing that this alloca is promotable, we know that it's safe to kill all
// instructions except for load and store.
+ // Determine the type used by loads/stores on this alloca. Per
+ // isAllocaPromotable, all loads/stores must use the same type, and GEP/
+ // bitcast/addrspacecast derived pointers cannot have load/store users, so
+ // loads/stores are always direct users of the alloca.
+ Type *PromotedType = nullptr;
+ for (User *U : AI->users()) {
+ if (auto *LI = dyn_cast<LoadInst>(U)) {
+ PromotedType = LI->getType();
+ break;
+ }
+ if (auto *SI = dyn_cast<StoreInst>(U)) {
+ PromotedType = SI->getValueOperand()->getType();
+ break;
+ }
+ }
+
for (Use &U : llvm::make_early_inc_range(AI->uses())) {
Instruction *I = cast<Instruction>(U.getUser());
if (isa<LoadInst>(I) || isa<StoreInst>(I))
@@ -550,14 +566,17 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
}
}
- // Same as above for lifetime intrinsics directly on the alloca.
- if (auto *II = dyn_cast<IntrinsicInst>(I))
- if (II->isLifetimeStartOrEnd()) {
- auto *Store = new StoreInst(UndefValue::get(AI->getAllocatedType()), AI,
- /*isVolatile=*/false, AI->getAlign(),
- I->getIterator());
- Store->setDebugLoc(II->getDebugLoc());
- }
+ // Same as above for lifetime intrinsics directly on the alloca. If the
+ // alloca has no load/store users, PromotedType is null and the alloca will
+ // be deleted as dead, so no store is needed.
+ if (PromotedType)
+ if (auto *II = dyn_cast<IntrinsicInst>(I))
+ if (II->isLifetimeStartOrEnd()) {
+ auto *Store = new StoreInst(UndefValue::get(PromotedType), AI,
+ /*isVolatile=*/false, AI->getAlign(),
+ I->getIterator());
+ Store->setDebugLoc(II->getDebugLoc());
+ }
I->eraseFromParent();
}
More information about the llvm-commits
mailing list