[llvm] [AMDGPU] Invalidate uniformity info after intrinsic combine (PR #222357)
Yaxun Liu via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 18:10:31 PDT 2026
https://github.com/yxsamliu updated https://github.com/llvm/llvm-project/pull/222357
>From 89270153228f1cd2628cbdfbfe3ceb1717960e2c Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Wed, 9 Sep 2026 09:51:05 -0400
Subject: [PATCH 1/4] [AMDGPU] Invalidate uniformity info after intrinsic
combine
The uniform intrinsic combine pass changed IR while preserving
uniformity information. Its result keeps references to cycle
information. Those references could be invalidated and later accessed
by another pass.
Invalidate uniformity information after a change. Before erasing an
instruction, remove it from the analysis state used by the pass. Also
report each erasure as a change so invalidation always runs.
---
.../AMDGPU/AMDGPUUniformIntrinsicCombine.cpp | 18 +++--
...intrinsic-combine-analysis-invalidation.ll | 67 +++++++++++++++++++
2 files changed, 78 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index 6201a60053369..02826f96ecbc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -50,8 +50,7 @@ isDivergentUseWithNew(const Use &U, const UniformityInfo &UI,
}
/// Optimizes uniform intrinsics calls if their operand can be proven uniform.
-static bool optimizeUniformIntrinsic(IntrinsicInst &II,
- const UniformityInfo &UI,
+static bool optimizeUniformIntrinsic(IntrinsicInst &II, UniformityInfo &UI,
ValueMap<const Value *, bool> &Tracker) {
llvm::Intrinsic::ID IID = II.getIntrinsicID();
/// We deliberately do not simplify readfirstlane with a uniform argument, so
@@ -65,6 +64,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
return false;
LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');
II.replaceAllUsesWith(Src);
+ UI.forgetValue(&II);
II.eraseFromParent();
return true;
}
@@ -100,8 +100,11 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
}
// Erase the intrinsic if it has no remaining uses.
- if (II.use_empty())
+ if (II.use_empty()) {
+ UI.forgetValue(&II);
II.eraseFromParent();
+ Changed = true;
+ }
return Changed;
}
case Intrinsic::amdgcn_wave_shuffle: {
@@ -111,6 +114,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
// Like with readlane, if Value is uniform then just propagate it
if (!isDivergentUseWithNew(Val, UI, Tracker)) {
II.replaceAllUsesWith(Val);
+ UI.forgetValue(&II);
II.eraseFromParent();
return true;
}
@@ -133,7 +137,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
/// Iterates over intrinsic calls in the Function to optimize.
-static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
+static bool runUniformIntrinsicCombine(Function &F, UniformityInfo &UI) {
bool IsChanged = false;
ValueMap<const Value *, bool> Tracker;
@@ -149,12 +153,12 @@ static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
PreservedAnalyses
AMDGPUUniformIntrinsicCombinePass::run(Function &F,
FunctionAnalysisManager &AM) {
- const auto &UI = AM.getResult<UniformityInfoAnalysis>(F);
+ UniformityInfo &UI = AM.getResult<UniformityInfoAnalysis>(F);
if (!runUniformIntrinsicCombine(F, UI))
return PreservedAnalyses::all();
PreservedAnalyses PA;
- PA.preserve<UniformityInfoAnalysis>();
+ PA.preserveSet<CFGAnalyses>();
return PA;
}
@@ -181,7 +185,7 @@ char &llvm::AMDGPUUniformIntrinsicCombineLegacyPassID =
bool AMDGPUUniformIntrinsicCombineLegacy::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
- const UniformityInfo &UI =
+ UniformityInfo &UI =
getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
return runUniformIntrinsicCombine(F, UI);
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
new file mode 100644
index 0000000000000..d55c89e0a18b4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
@@ -0,0 +1,67 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa \
+; RUN: -passes='amdgpu-uniform-intrinsic-combine,reassociate' \
+; RUN: -verify-analysis-invalidation -disable-output < %s
+
+; Reassociate must not use uniformity information that refers to invalidated
+; cycle information after the combine changes the IR.
+
+declare i64 @llvm.amdgcn.ballot.i64(i1)
+
+; The pass must report a change when erasing an already unused ballot.
+define amdgpu_kernel void @erase_unused_ballot() {
+entry:
+ %mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @invalidate_uniformity_info() {
+entry:
+ %base = mul i32 0, 0
+ %factor = zext i16 0 to i32
+ %product = mul i32 %base, %factor
+ %sum = or i32 0, %product
+ br label %dispatch
+
+dispatch:
+ br label %select
+
+select:
+ switch i32 0, label %default [
+ i32 0, label %ballot
+ i32 1, label %left
+ i32 2, label %right
+ ]
+
+ballot:
+ %mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+ %none.active = icmp eq i64 %mask, 0
+ br label %unreachable.block
+
+unreachable.block:
+ unreachable
+
+left:
+ br label %left.next
+
+left.next:
+ br label %left.merge
+
+left.merge:
+ br label %exit
+
+right:
+ br label %right.next
+
+right.next:
+ br label %right.merge
+
+right.merge:
+ br label %exit
+
+default:
+ ret void
+
+exit:
+ ret void
+}
>From bac88a75332b7482ee96cd17f1c6fddbedccc62c Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 10 Sep 2026 08:27:11 -0400
Subject: [PATCH 2/4] [AMDGPU] Check uniform intrinsic combine output
---
...-uniform-intrinsic-combine-analysis-invalidation.ll | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
index d55c89e0a18b4..738441142f2a3 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
@@ -1,5 +1,7 @@
; REQUIRES: asserts
; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa \
+; RUN: -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa \
; RUN: -passes='amdgpu-uniform-intrinsic-combine,reassociate' \
; RUN: -verify-analysis-invalidation -disable-output < %s
@@ -10,12 +12,20 @@ declare i64 @llvm.amdgcn.ballot.i64(i1)
; The pass must report a change when erasing an already unused ballot.
define amdgpu_kernel void @erase_unused_ballot() {
+; CHECK-LABEL: define amdgpu_kernel void @erase_unused_ballot()
+; CHECK-NOT: call i64 @llvm.amdgcn.ballot
+; CHECK: ret void
entry:
%mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
ret void
}
define amdgpu_kernel void @invalidate_uniformity_info() {
+; CHECK-LABEL: define amdgpu_kernel void @invalidate_uniformity_info()
+; CHECK: ballot:
+; CHECK-NEXT: %[[MASK:.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+; CHECK-NEXT: %[[NOT:.*]] = xor i1 false, true
+; CHECK-NEXT: %none.active = icmp eq i64 %[[MASK]], 0
entry:
%base = mul i32 0, 0
%factor = zext i16 0 to i32
>From e0f136952867a03ef7d910a0ddf0c8b1753fa4a3 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 10 Sep 2026 15:41:59 -0400
Subject: [PATCH 3/4] [AMDGPU] Simplify uniformity invalidation test
---
...pu-uniform-intrinsic-combine-analysis-invalidation.ll | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
index 738441142f2a3..65f86be03b867 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
@@ -1,15 +1,10 @@
; REQUIRES: asserts
-; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa \
-; RUN: -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
-; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa \
-; RUN: -passes='amdgpu-uniform-intrinsic-combine,reassociate' \
-; RUN: -verify-analysis-invalidation -disable-output < %s
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes='amdgpu-uniform-intrinsic-combine,reassociate' -verify-analysis-invalidation -disable-output < %s
; Reassociate must not use uniformity information that refers to invalidated
; cycle information after the combine changes the IR.
-declare i64 @llvm.amdgcn.ballot.i64(i1)
-
; The pass must report a change when erasing an already unused ballot.
define amdgpu_kernel void @erase_unused_ballot() {
; CHECK-LABEL: define amdgpu_kernel void @erase_unused_ballot()
>From 9ec07bc733b9a9f0886b43627f0c970f1ea741cd Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <yaxun.liu at amd.com>
Date: Thu, 10 Sep 2026 21:09:49 -0400
Subject: [PATCH 4/4] [AMDGPU] Autogenerate uniform intrinsic combine checks
---
...intrinsic-combine-analysis-invalidation.ll | 58 +++++++++++++++----
1 file changed, 47 insertions(+), 11 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
index 65f86be03b867..9e65aa610defa 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; REQUIRES: asserts
; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes='amdgpu-uniform-intrinsic-combine,reassociate' -verify-analysis-invalidation -disable-output < %s
@@ -7,20 +8,55 @@
; The pass must report a change when erasing an already unused ballot.
define amdgpu_kernel void @erase_unused_ballot() {
-; CHECK-LABEL: define amdgpu_kernel void @erase_unused_ballot()
-; CHECK-NOT: call i64 @llvm.amdgcn.ballot
-; CHECK: ret void
+; CHECK-LABEL: define amdgpu_kernel void @erase_unused_ballot() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: ret void
+;
entry:
%mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
ret void
}
define amdgpu_kernel void @invalidate_uniformity_info() {
-; CHECK-LABEL: define amdgpu_kernel void @invalidate_uniformity_info()
-; CHECK: ballot:
-; CHECK-NEXT: %[[MASK:.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 false)
-; CHECK-NEXT: %[[NOT:.*]] = xor i1 false, true
-; CHECK-NEXT: %none.active = icmp eq i64 %[[MASK]], 0
+; CHECK-LABEL: define amdgpu_kernel void @invalidate_uniformity_info() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE:%.*]] = mul i32 0, 0
+; CHECK-NEXT: [[FACTOR:%.*]] = zext i16 0 to i32
+; CHECK-NEXT: [[PRODUCT:%.*]] = mul i32 [[BASE]], [[FACTOR]]
+; CHECK-NEXT: [[SUM:%.*]] = or i32 0, [[PRODUCT]]
+; CHECK-NEXT: br label %[[DISPATCH:.*]]
+; CHECK: [[DISPATCH]]:
+; CHECK-NEXT: br label %[[SELECT:.*]]
+; CHECK: [[SELECT]]:
+; CHECK-NEXT: switch i32 0, label %[[DEFAULT:.*]] [
+; CHECK-NEXT: i32 0, label %[[BALLOT:.*]]
+; CHECK-NEXT: i32 1, label %[[LEFT:.*]]
+; CHECK-NEXT: i32 2, label %[[RIGHT:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[BALLOT]]:
+; CHECK-NEXT: [[MASK:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+; CHECK-NEXT: [[TMP0:%.*]] = xor i1 false, true
+; CHECK-NEXT: [[NONE_ACTIVE:%.*]] = icmp eq i64 [[MASK]], 0
+; CHECK-NEXT: br label %[[UNREACHABLE_BLOCK:.*]]
+; CHECK: [[UNREACHABLE_BLOCK]]:
+; CHECK-NEXT: unreachable
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[LEFT_NEXT:.*]]
+; CHECK: [[LEFT_NEXT]]:
+; CHECK-NEXT: br label %[[LEFT_MERGE:.*]]
+; CHECK: [[LEFT_MERGE]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[RIGHT_NEXT:.*]]
+; CHECK: [[RIGHT_NEXT]]:
+; CHECK-NEXT: br label %[[RIGHT_MERGE:.*]]
+; CHECK: [[RIGHT_MERGE]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[DEFAULT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
%base = mul i32 0, 0
%factor = zext i16 0 to i32
@@ -33,9 +69,9 @@ dispatch:
select:
switch i32 0, label %default [
- i32 0, label %ballot
- i32 1, label %left
- i32 2, label %right
+ i32 0, label %ballot
+ i32 1, label %left
+ i32 2, label %right
]
ballot:
More information about the llvm-commits
mailing list