[llvm] [PromoteMemToReg] Hoist removeIntrinsicUsers into a pre-pass before promotion (PR #224617)
Marcin Radomski via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 01:56:09 PDT 2026
https://github.com/dextero updated https://github.com/llvm/llvm-project/pull/224617
>From 373fbfd705bc0a44dceae08ec9b92868cc7bdc58 Mon Sep 17 00:00:00 2001
From: Marcin Radomski <dextero at google.com>
Date: Fri, 18 Sep 2026 11:07:50 +0000
Subject: [PATCH 1/2] [PromoteMemToReg] Hoist removeIntrinsicUsers into a
pre-pass before promotion
Calling removeIntrinsicUsers(AI) inside the per-alloca promotion loop
in PromoteMem2Reg::run() inserts StoreInst(undef) instructions into
basic blocks after LargeBlockInfo (LBI) has already cached instruction
indices for those blocks during earlier iterations. Every subsequent
LBI.getInstructionIndex query on newly inserted StoreInst(undef)
instructions misses in LBI.InstNumbers and triggers a full O(|BB|)
basic block rescan, resulting in O(K * |BB|) compilation time blowup
in promoteSingleBlockAlloca when a basic block contains K scoped
allocas.
Hoist removeIntrinsicUsers(AI) into a pre-pass loop over all Allocas
upfront in PromoteMem2Reg::run() before the main AllocaNum promotion
loop begins so all synthetic StoreInst(undef) instructions are
inserted prior to LBI caching.
This improves the compile time of some CUDA kernels [1] by ~9x.
[1] https://github.com/llvm/llvm-project/pull/191909#issuecomment-5569614846
Assisted-by: Gemini
---
.../Utils/PromoteMemoryToRegister.cpp | 9 +-
.../lifetime-prepass-multiple-allocas.ll | 82 +++++++++++++++++++
2 files changed, 87 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 21ddc78f1469b..900593fb8bdce 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -810,14 +810,15 @@ void PromoteMem2Reg::run() {
NoSignedZeros = F.getFnAttribute("no-signed-zeros-fp-math").getValueAsBool();
- for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); ++AllocaNum) {
- AllocaInst *AI = Allocas[AllocaNum];
-
+ for (AllocaInst *AI : Allocas) {
assert(isAllocaPromotable(AI) && "Cannot promote non-promotable alloca!");
assert(AI->getParent()->getParent() == &F &&
"All allocas should be in the same function, which is same as DF!");
-
removeIntrinsicUsers(AI);
+ }
+
+ for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); ++AllocaNum) {
+ AllocaInst *AI = Allocas[AllocaNum];
if (AI->use_empty()) {
// If there are no uses of the alloca, just delete it now.
diff --git a/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll b/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
new file mode 100644
index 0000000000000..40df623ae2bae
--- /dev/null
+++ b/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
@@ -0,0 +1,82 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=mem2reg -S < %s | FileCheck %s
+
+declare void @use(i32)
+declare void @llvm.assume(i1)
+
+; Verify that multiple single-block allocas with lifetime markers in the same
+; block are promoted accurately when removeIntrinsicUsers runs as a pre-pass
+; before LargeBlockInfo caches instruction indices.
+define i32 @multiple_single_block_allocas_with_lifetime(i32 %x, i32 %y) {
+; CHECK-LABEL: define i32 @multiple_single_block_allocas_with_lifetime(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ call void @llvm.lifetime.start.p0(ptr %a)
+ store i32 %x, ptr %a, align 4
+ %va = load i32, ptr %a, align 4
+ call void @llvm.lifetime.end.p0(ptr %a)
+ call void @llvm.lifetime.start.p0(ptr %b)
+ store i32 %y, ptr %b, align 4
+ %vb = load i32, ptr %b, align 4
+ call void @llvm.lifetime.end.p0(ptr %b)
+ %sum = add i32 %va, %vb
+ ret i32 %sum
+}
+
+; Verify that lifetime.start and lifetime.end on an addrspace(5) alloca
+; alongside zero-index GEP droppable users insert store undef in the pre-pass,
+; breaking loop-carried back-edge dependencies.
+define void @lifetime_addrspace5_and_gep_in_loop(i32 %val, i1 %c1, i1 %c2) {
+; CHECK-LABEL: define void @lifetime_addrspace5_and_gep_in_loop(
+; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(ptr addrspace(5) poison, i64 4) ]
+; CHECK-NEXT: br i1 [[C1]], label %[[INIT:.*]], label %[[SKIP:.*]]
+; CHECK: [[INIT]]:
+; CHECK-NEXT: br label %[[READ:.*]]
+; CHECK: [[SKIP]]:
+; CHECK-NEXT: br label %[[READ]]
+; CHECK: [[READ]]:
+; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ [[VAL]], %[[INIT]] ], [ undef, %[[SKIP]] ]
+; CHECK-NEXT: call void @use(i32 [[A_0]])
+; CHECK-NEXT: br label %[[CLEANUP:.*]]
+; CHECK: [[CLEANUP]]:
+; CHECK-NEXT: br i1 [[C2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = alloca i32, align 4, addrspace(5)
+ br label %loop
+
+loop:
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %a)
+ %gep = getelementptr inbounds i32, ptr addrspace(5) %a, i64 0
+ call void @llvm.assume(i1 true) ["align"(ptr addrspace(5) %gep, i64 4)]
+ br i1 %c1, label %init, label %skip
+
+init:
+ store i32 %val, ptr addrspace(5) %a, align 4
+ br label %read
+
+skip:
+ br label %read
+
+read:
+ %v = load i32, ptr addrspace(5) %a, align 4
+ call void @use(i32 %v)
+ br label %cleanup
+
+cleanup:
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %a)
+ br i1 %c2, label %loop, label %exit
+
+exit:
+ ret void
+}
>From e99fc8e1dbaf63ef3d3bb026c8eee09360819b7b Mon Sep 17 00:00:00 2001
From: Marcin Radomski <dextero at google.com>
Date: Thu, 17 Sep 2026 14:32:20 +0000
Subject: [PATCH 2/2] [PromoteMemToReg] Add comment, move tests to existing
files
---
.../Utils/PromoteMemoryToRegister.cpp | 4 +
.../Transforms/Mem2Reg/ignore-droppable.ll | 55 +++++++++++++
.../Transforms/Mem2Reg/ignore-lifetime.ll | 23 ++++++
.../lifetime-prepass-multiple-allocas.ll | 82 -------------------
4 files changed, 82 insertions(+), 82 deletions(-)
delete mode 100644 llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index 900593fb8bdce..157c3de56be02 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -810,6 +810,10 @@ void PromoteMem2Reg::run() {
NoSignedZeros = F.getFnAttribute("no-signed-zeros-fp-math").getValueAsBool();
+ // removeIntrinsicUsers inserts StoreInst(undef). A cache miss on
+ // LBI.getInstructionIndex lookup causes a full O(|BB|) basic block rescan.
+ // Doing all inserts in advance lets us capture all the new instructions in
+ // just a single rescan.
for (AllocaInst *AI : Allocas) {
assert(isAllocaPromotable(AI) && "Cannot promote non-promotable alloca!");
assert(AI->getParent()->getParent() == &F &&
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
index a876319281b17..e5236c7b3d96d 100644
--- a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
+++ b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
@@ -78,3 +78,58 @@ define void @positive_mixed_assume_uses() {
call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 2), "nonnull"(ptr %A)]
ret void
}
+
+declare void @use(i32)
+
+; Verify that lifetime.start and lifetime.end on an addrspace(5) alloca
+; alongside zero-index GEP droppable users insert store undef in the pre-pass,
+; breaking loop-carried back-edge dependencies.
+define void @lifetime_addrspace5_and_gep_in_loop(i32 %val, i1 %c1, i1 %c2) {
+; CHECK-LABEL: @lifetime_addrspace5_and_gep_in_loop(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(ptr addrspace(5) poison, i64 4) ]
+; CHECK-NEXT: br i1 [[C1:%.*]], label [[INIT:%.*]], label [[SKIP:%.*]]
+; CHECK: init:
+; CHECK-NEXT: br label [[READ:%.*]]
+; CHECK: skip:
+; CHECK-NEXT: br label [[READ]]
+; CHECK: read:
+; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ [[VAL:%.*]], [[INIT]] ], [ undef, [[SKIP]] ]
+; CHECK-NEXT: call void @use(i32 [[A_0]])
+; CHECK-NEXT: br label [[CLEANUP:%.*]]
+; CHECK: cleanup:
+; CHECK-NEXT: br i1 [[C2:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = alloca i32, align 4, addrspace(5)
+ br label %loop
+
+loop:
+ call void @llvm.lifetime.start.p5(ptr addrspace(5) %a)
+ %gep = getelementptr inbounds i32, ptr addrspace(5) %a, i64 0
+ call void @llvm.assume(i1 true) ["align"(ptr addrspace(5) %gep, i64 4)]
+ br i1 %c1, label %init, label %skip
+
+init:
+ store i32 %val, ptr addrspace(5) %a, align 4
+ br label %read
+
+skip:
+ br label %read
+
+read:
+ %v = load i32, ptr addrspace(5) %a, align 4
+ call void @use(i32 %v)
+ br label %cleanup
+
+cleanup:
+ call void @llvm.lifetime.end.p5(ptr addrspace(5) %a)
+ br i1 %c2, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
index 510fb2b8638e0..c0cd14fc7c5df 100644
--- a/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
+++ b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
@@ -22,3 +22,26 @@ define void @test2() {
call void @llvm.lifetime.end.p0(ptr %A)
ret void
}
+
+; Verify that multiple single-block allocas with lifetime markers in the same
+; block are promoted accurately when removeIntrinsicUsers runs as a pre-pass
+; before LargeBlockInfo caches instruction indices.
+define i32 @multiple_single_block_allocas_with_lifetime(i32 %x, i32 %y) {
+; CHECK-LABEL: define i32 @multiple_single_block_allocas_with_lifetime(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X]], [[Y]]
+; CHECK-NEXT: ret i32 [[SUM]]
+;
+ %a = alloca i32, align 4
+ %b = alloca i32, align 4
+ call void @llvm.lifetime.start.p0(ptr %a)
+ store i32 %x, ptr %a, align 4
+ %va = load i32, ptr %a, align 4
+ call void @llvm.lifetime.end.p0(ptr %a)
+ call void @llvm.lifetime.start.p0(ptr %b)
+ store i32 %y, ptr %b, align 4
+ %vb = load i32, ptr %b, align 4
+ call void @llvm.lifetime.end.p0(ptr %b)
+ %sum = add i32 %va, %vb
+ ret i32 %sum
+}
diff --git a/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll b/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
deleted file mode 100644
index 40df623ae2bae..0000000000000
--- a/llvm/test/Transforms/Mem2Reg/lifetime-prepass-multiple-allocas.ll
+++ /dev/null
@@ -1,82 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=mem2reg -S < %s | FileCheck %s
-
-declare void @use(i32)
-declare void @llvm.assume(i1)
-
-; Verify that multiple single-block allocas with lifetime markers in the same
-; block are promoted accurately when removeIntrinsicUsers runs as a pre-pass
-; before LargeBlockInfo caches instruction indices.
-define i32 @multiple_single_block_allocas_with_lifetime(i32 %x, i32 %y) {
-; CHECK-LABEL: define i32 @multiple_single_block_allocas_with_lifetime(
-; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
-; CHECK-NEXT: [[SUM:%.*]] = add i32 [[X]], [[Y]]
-; CHECK-NEXT: ret i32 [[SUM]]
-;
- %a = alloca i32, align 4
- %b = alloca i32, align 4
- call void @llvm.lifetime.start.p0(ptr %a)
- store i32 %x, ptr %a, align 4
- %va = load i32, ptr %a, align 4
- call void @llvm.lifetime.end.p0(ptr %a)
- call void @llvm.lifetime.start.p0(ptr %b)
- store i32 %y, ptr %b, align 4
- %vb = load i32, ptr %b, align 4
- call void @llvm.lifetime.end.p0(ptr %b)
- %sum = add i32 %va, %vb
- ret i32 %sum
-}
-
-; Verify that lifetime.start and lifetime.end on an addrspace(5) alloca
-; alongside zero-index GEP droppable users insert store undef in the pre-pass,
-; breaking loop-carried back-edge dependencies.
-define void @lifetime_addrspace5_and_gep_in_loop(i32 %val, i1 %c1, i1 %c2) {
-; CHECK-LABEL: define void @lifetime_addrspace5_and_gep_in_loop(
-; CHECK-SAME: i32 [[VAL:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[LOOP:.*]]
-; CHECK: [[LOOP]]:
-; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(ptr addrspace(5) poison, i64 4) ]
-; CHECK-NEXT: br i1 [[C1]], label %[[INIT:.*]], label %[[SKIP:.*]]
-; CHECK: [[INIT]]:
-; CHECK-NEXT: br label %[[READ:.*]]
-; CHECK: [[SKIP]]:
-; CHECK-NEXT: br label %[[READ]]
-; CHECK: [[READ]]:
-; CHECK-NEXT: [[A_0:%.*]] = phi i32 [ [[VAL]], %[[INIT]] ], [ undef, %[[SKIP]] ]
-; CHECK-NEXT: call void @use(i32 [[A_0]])
-; CHECK-NEXT: br label %[[CLEANUP:.*]]
-; CHECK: [[CLEANUP]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[LOOP]], label %[[EXIT:.*]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- %a = alloca i32, align 4, addrspace(5)
- br label %loop
-
-loop:
- call void @llvm.lifetime.start.p5(ptr addrspace(5) %a)
- %gep = getelementptr inbounds i32, ptr addrspace(5) %a, i64 0
- call void @llvm.assume(i1 true) ["align"(ptr addrspace(5) %gep, i64 4)]
- br i1 %c1, label %init, label %skip
-
-init:
- store i32 %val, ptr addrspace(5) %a, align 4
- br label %read
-
-skip:
- br label %read
-
-read:
- %v = load i32, ptr addrspace(5) %a, align 4
- call void @use(i32 %v)
- br label %cleanup
-
-cleanup:
- call void @llvm.lifetime.end.p5(ptr addrspace(5) %a)
- br i1 %c2, label %loop, label %exit
-
-exit:
- ret void
-}
More information about the llvm-commits
mailing list