[llvm] 4267061 - [AMDGPU] Invalidate uniformity info after intrinsic combine (#222357)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 20:04:21 PDT 2026
Author: Yaxun (Sam) Liu
Date: 2026-09-10T23:04:16-04:00
New Revision: 426706131e7300fe67f4dac5ccc744548605f394
URL: https://github.com/llvm/llvm-project/commit/426706131e7300fe67f4dac5ccc744548605f394
DIFF: https://github.com/llvm/llvm-project/commit/426706131e7300fe67f4dac5ccc744548605f394.diff
LOG: [AMDGPU] Invalidate uniformity info after intrinsic combine (#222357)
The uniform intrinsic combine pass changed IR while preserving
uniformity information. Its result keeps references to cycle
information. Those references could be invalidated and later accessed
by another pass.
Invalidate uniformity information after a change. Before erasing an
instruction, remove it from the analysis state used by the pass. Also
report each erasure as a change so invalidation always runs.
Added:
llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index 6201a60053369..02826f96ecbc7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -50,8 +50,7 @@ isDivergentUseWithNew(const Use &U, const UniformityInfo &UI,
}
/// Optimizes uniform intrinsics calls if their operand can be proven uniform.
-static bool optimizeUniformIntrinsic(IntrinsicInst &II,
- const UniformityInfo &UI,
+static bool optimizeUniformIntrinsic(IntrinsicInst &II, UniformityInfo &UI,
ValueMap<const Value *, bool> &Tracker) {
llvm::Intrinsic::ID IID = II.getIntrinsicID();
/// We deliberately do not simplify readfirstlane with a uniform argument, so
@@ -65,6 +64,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
return false;
LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');
II.replaceAllUsesWith(Src);
+ UI.forgetValue(&II);
II.eraseFromParent();
return true;
}
@@ -100,8 +100,11 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
}
// Erase the intrinsic if it has no remaining uses.
- if (II.use_empty())
+ if (II.use_empty()) {
+ UI.forgetValue(&II);
II.eraseFromParent();
+ Changed = true;
+ }
return Changed;
}
case Intrinsic::amdgcn_wave_shuffle: {
@@ -111,6 +114,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
// Like with readlane, if Value is uniform then just propagate it
if (!isDivergentUseWithNew(Val, UI, Tracker)) {
II.replaceAllUsesWith(Val);
+ UI.forgetValue(&II);
II.eraseFromParent();
return true;
}
@@ -133,7 +137,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
/// Iterates over intrinsic calls in the Function to optimize.
-static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
+static bool runUniformIntrinsicCombine(Function &F, UniformityInfo &UI) {
bool IsChanged = false;
ValueMap<const Value *, bool> Tracker;
@@ -149,12 +153,12 @@ static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
PreservedAnalyses
AMDGPUUniformIntrinsicCombinePass::run(Function &F,
FunctionAnalysisManager &AM) {
- const auto &UI = AM.getResult<UniformityInfoAnalysis>(F);
+ UniformityInfo &UI = AM.getResult<UniformityInfoAnalysis>(F);
if (!runUniformIntrinsicCombine(F, UI))
return PreservedAnalyses::all();
PreservedAnalyses PA;
- PA.preserve<UniformityInfoAnalysis>();
+ PA.preserveSet<CFGAnalyses>();
return PA;
}
@@ -181,7 +185,7 @@ char &llvm::AMDGPUUniformIntrinsicCombineLegacyPassID =
bool AMDGPUUniformIntrinsicCombineLegacy::runOnFunction(Function &F) {
if (skipFunction(F))
return false;
- const UniformityInfo &UI =
+ UniformityInfo &UI =
getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
return runUniformIntrinsicCombine(F, UI);
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
new file mode 100644
index 0000000000000..9e65aa610defa
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-analysis-invalidation.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; REQUIRES: asserts
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
+; RUN: opt -mtriple=amdgpu10.10-amd-amdhsa -passes='amdgpu-uniform-intrinsic-combine,reassociate' -verify-analysis-invalidation -disable-output < %s
+
+; Reassociate must not use uniformity information that refers to invalidated
+; cycle information after the combine changes the IR.
+
+; The pass must report a change when erasing an already unused ballot.
+define amdgpu_kernel void @erase_unused_ballot() {
+; CHECK-LABEL: define amdgpu_kernel void @erase_unused_ballot() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: ret void
+;
+entry:
+ %mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+ ret void
+}
+
+define amdgpu_kernel void @invalidate_uniformity_info() {
+; CHECK-LABEL: define amdgpu_kernel void @invalidate_uniformity_info() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE:%.*]] = mul i32 0, 0
+; CHECK-NEXT: [[FACTOR:%.*]] = zext i16 0 to i32
+; CHECK-NEXT: [[PRODUCT:%.*]] = mul i32 [[BASE]], [[FACTOR]]
+; CHECK-NEXT: [[SUM:%.*]] = or i32 0, [[PRODUCT]]
+; CHECK-NEXT: br label %[[DISPATCH:.*]]
+; CHECK: [[DISPATCH]]:
+; CHECK-NEXT: br label %[[SELECT:.*]]
+; CHECK: [[SELECT]]:
+; CHECK-NEXT: switch i32 0, label %[[DEFAULT:.*]] [
+; CHECK-NEXT: i32 0, label %[[BALLOT:.*]]
+; CHECK-NEXT: i32 1, label %[[LEFT:.*]]
+; CHECK-NEXT: i32 2, label %[[RIGHT:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[BALLOT]]:
+; CHECK-NEXT: [[MASK:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+; CHECK-NEXT: [[TMP0:%.*]] = xor i1 false, true
+; CHECK-NEXT: [[NONE_ACTIVE:%.*]] = icmp eq i64 [[MASK]], 0
+; CHECK-NEXT: br label %[[UNREACHABLE_BLOCK:.*]]
+; CHECK: [[UNREACHABLE_BLOCK]]:
+; CHECK-NEXT: unreachable
+; CHECK: [[LEFT]]:
+; CHECK-NEXT: br label %[[LEFT_NEXT:.*]]
+; CHECK: [[LEFT_NEXT]]:
+; CHECK-NEXT: br label %[[LEFT_MERGE:.*]]
+; CHECK: [[LEFT_MERGE]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[RIGHT]]:
+; CHECK-NEXT: br label %[[RIGHT_NEXT:.*]]
+; CHECK: [[RIGHT_NEXT]]:
+; CHECK-NEXT: br label %[[RIGHT_MERGE:.*]]
+; CHECK: [[RIGHT_MERGE]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[DEFAULT]]:
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %base = mul i32 0, 0
+ %factor = zext i16 0 to i32
+ %product = mul i32 %base, %factor
+ %sum = or i32 0, %product
+ br label %dispatch
+
+dispatch:
+ br label %select
+
+select:
+ switch i32 0, label %default [
+ i32 0, label %ballot
+ i32 1, label %left
+ i32 2, label %right
+ ]
+
+ballot:
+ %mask = call i64 @llvm.amdgcn.ballot.i64(i1 false)
+ %none.active = icmp eq i64 %mask, 0
+ br label %unreachable.block
+
+unreachable.block:
+ unreachable
+
+left:
+ br label %left.next
+
+left.next:
+ br label %left.merge
+
+left.merge:
+ br label %exit
+
+right:
+ br label %right.next
+
+right.next:
+ br label %right.merge
+
+right.merge:
+ br label %exit
+
+default:
+ ret void
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list