[llvm] [AMDGPU] Fix wrong uniformity for ballot NOT in UniformIntrinsicCombine (PR #220009)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 1 03:49:40 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/220009
>From b37baf220656d1b60d79790f0434ce8faba121f7 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Mon, 31 Aug 2026 17:38:37 +0200
Subject: [PATCH 1/2] [AMDGPU] Fix wrong uniformity for ballot NOT in
UniformIntrinsicCombine
The pass always treated the NOT it builds from a ballot as uniform
If the NOT crossed a loop, it could actually be divergent and a later use of it got folded as if it were not
---
.../AMDGPU/AMDGPUUniformIntrinsicCombine.cpp | 29 +-
.../amdgpu-uniform-temporal-divergence.ll | 321 ++++++++++++++++++
2 files changed, 333 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index 6b68c25825d7b..b0e7c91c1dd26 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -41,21 +41,16 @@ using namespace llvm;
using namespace llvm::AMDGPU;
using namespace llvm::PatternMatch;
-/// Wrapper for querying uniformity info that first checks locally tracked
-/// instructions.
-static bool
-isDivergentUseWithNew(const Use &U, const UniformityInfo &UI,
- const ValueMap<const Value *, bool> &Tracker) {
- Value *V = U.get();
- if (auto It = Tracker.find(V); It != Tracker.end())
- return !It->second; // divergent if marked false
- return UI.isDivergentAtUse(U);
+/// Divergent per UI, or newly created (guards against address reuse).
+static bool isDivergentUse(const Use &U, const UniformityInfo &UI,
+ const SmallPtrSetImpl<const Value *> &NewInsts) {
+ return NewInsts.contains(U.get()) || UI.isDivergentAtUse(U);
}
/// Optimizes uniform intrinsics calls if their operand can be proven uniform.
static bool optimizeUniformIntrinsic(IntrinsicInst &II,
const UniformityInfo &UI,
- ValueMap<const Value *, bool> &Tracker) {
+ SmallPtrSetImpl<const Value *> &NewInsts) {
llvm::Intrinsic::ID IID = II.getIntrinsicID();
/// We deliberately do not simplify readfirstlane with a uniform argument, so
/// that frontends can use it to force a copy to SGPR and thereby prevent the
@@ -64,7 +59,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readlane: {
Value *Src = II.getArgOperand(0);
- if (isDivergentUseWithNew(II.getOperandUse(0), UI, Tracker))
+ if (isDivergentUse(II.getOperandUse(0), UI, NewInsts))
return false;
LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');
II.replaceAllUsesWith(Src);
@@ -73,7 +68,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
case Intrinsic::amdgcn_ballot: {
Value *Src = II.getArgOperand(0);
- if (isDivergentUseWithNew(II.getOperandUse(0), UI, Tracker))
+ if (isDivergentUse(II.getOperandUse(0), UI, NewInsts))
return false;
LLVM_DEBUG(dbgs() << "Found uniform ballot intrinsic: " << II << '\n');
@@ -89,7 +84,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
// Case: (icmp eq %ballot, 0) -> xor %ballot_arg, 1
Instruction *NotOp =
BinaryOperator::CreateNot(Src, "", ICmp->getIterator());
- Tracker[NotOp] = true; // NOT preserves uniformity
+ NewInsts.insert(NotOp);
LLVM_DEBUG(dbgs() << "Replacing ICMP_EQ: " << *NotOp << '\n');
ICmp->replaceAllUsesWith(NotOp);
Changed = true;
@@ -112,14 +107,14 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
Use &Idx = II.getOperandUse(1);
// Like with readlane, if Value is uniform then just propagate it
- if (!isDivergentUseWithNew(Val, UI, Tracker)) {
+ if (!isDivergentUse(Val, UI, NewInsts)) {
II.replaceAllUsesWith(Val);
II.eraseFromParent();
return true;
}
// Otherwise, when Index is uniform, this is just a readlane operation
- if (isDivergentUseWithNew(Idx, UI, Tracker))
+ if (isDivergentUse(Idx, UI, NewInsts))
return false;
// The readlane intrinsic we want to call has the exact same function
@@ -138,13 +133,13 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
/// Iterates over intrinsic calls in the Function to optimize.
static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
bool IsChanged = false;
- ValueMap<const Value *, bool> Tracker;
+ SmallPtrSet<const Value *, 8> NewInsts;
for (Instruction &I : make_early_inc_range(instructions(F))) {
auto *II = dyn_cast<IntrinsicInst>(&I);
if (!II)
continue;
- IsChanged |= optimizeUniformIntrinsic(*II, UI, Tracker);
+ IsChanged |= optimizeUniformIntrinsic(*II, UI, NewInsts);
}
return IsChanged;
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
index 16312f8289339..77d88e3ab6f66 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
@@ -56,3 +56,324 @@ X:
declare i32 @llvm.amdgcn.workitem.id.x()
declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+
+define amdgpu_kernel void @temporal_divergence_ballot_chain(ptr addrspace(1) %out, ptr addrspace(1) %data) {
+; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_ballot_chain(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; PASS-CHECK-NEXT: [[ENTRY:.*]]:
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: br label %[[H:.*]]
+; PASS-CHECK: [[H]]:
+; PASS-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; PASS-CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[DATA]], i32 [[I]]
+; PASS-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
+; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
+; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
+; PASS-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; PASS-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; PASS-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; PASS-CHECK: [[EXIT]]:
+; PASS-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[TMP0]])
+; PASS-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; PASS-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; PASS-CHECK-NEXT: [[OG:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; PASS-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OG]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; COMB-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_ballot_chain(
+; COMB-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; COMB-CHECK-NEXT: [[ENTRY:.*]]:
+; COMB-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; COMB-CHECK-NEXT: br label %[[H:.*]]
+; COMB-CHECK: [[H]]:
+; COMB-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; COMB-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; COMB-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; COMB-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; COMB-CHECK: [[EXIT]]:
+; COMB-CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[I]] to i64
+; COMB-CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[DATA]], i64 [[TMP0]]
+; COMB-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
+; COMB-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; COMB-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; COMB-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; COMB-CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[TID]] to i64
+; COMB-CHECK-NEXT: [[OG:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[OUT]], i64 [[TMP1]]
+; COMB-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OG]], align 4
+; COMB-CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %H
+
+H:
+ %i = phi i32 [ 0, %entry ], [ %inc, %H ]
+ %gep = getelementptr i32, ptr addrspace(1) %data, i32 %i
+ %v = load i32, ptr addrspace(1) %gep
+ %x = icmp sgt i32 %v, 0
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
+ %none = icmp eq i32 %ballot, 0
+ %inc = add i32 %i, 1
+ %div.exitx = icmp ugt i32 %inc, %tid
+ br i1 %div.exitx, label %exit, label %H ; divergent branch
+
+exit:
+ %ballot2 = call i32 @llvm.amdgcn.ballot.i32(i1 %none)
+ %any = icmp ne i32 %ballot2, 0
+ %z = zext i1 %any to i32
+ %og = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %z, ptr addrspace(1) %og
+ ret void
+}
+
+define amdgpu_kernel void @temporal_divergence_ballot_user_outside(ptr addrspace(1) %out, ptr addrspace(1) %data) {
+; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_ballot_user_outside(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; PASS-CHECK-NEXT: [[ENTRY:.*]]:
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: br label %[[H:.*]]
+; PASS-CHECK: [[H]]:
+; PASS-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; PASS-CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[DATA]], i32 [[I]]
+; PASS-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
+; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; PASS-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; PASS-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; PASS-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; PASS-CHECK: [[EXIT]]:
+; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
+; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
+; PASS-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[TMP0]])
+; PASS-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; PASS-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; PASS-CHECK-NEXT: [[OG:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; PASS-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OG]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; COMB-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_ballot_user_outside(
+; COMB-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; COMB-CHECK-NEXT: [[ENTRY:.*]]:
+; COMB-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; COMB-CHECK-NEXT: br label %[[H:.*]]
+; COMB-CHECK: [[H]]:
+; COMB-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; COMB-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; COMB-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; COMB-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; COMB-CHECK: [[EXIT]]:
+; COMB-CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[I]] to i64
+; COMB-CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[DATA]], i64 [[TMP0]]
+; COMB-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
+; COMB-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; COMB-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; COMB-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; COMB-CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[TID]] to i64
+; COMB-CHECK-NEXT: [[OG:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[OUT]], i64 [[TMP1]]
+; COMB-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OG]], align 4
+; COMB-CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %H
+
+H:
+ %i = phi i32 [ 0, %entry ], [ %inc, %H ]
+ %gep = getelementptr i32, ptr addrspace(1) %data, i32 %i
+ %v = load i32, ptr addrspace(1) %gep
+ %x = icmp sgt i32 %v, 0
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
+ %inc = add i32 %i, 1
+ %div.exitx = icmp ugt i32 %inc, %tid
+ br i1 %div.exitx, label %exit, label %H ; divergent branch
+
+exit:
+ %none = icmp eq i32 %ballot, 0
+ %ballot2 = call i32 @llvm.amdgcn.ballot.i32(i1 %none)
+ %any = icmp ne i32 %ballot2, 0
+ %z = zext i1 %any to i32
+ %og = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %z, ptr addrspace(1) %og
+ ret void
+}
+
+define amdgpu_kernel void @temporal_divergence_readlane_derived(ptr addrspace(1) %out, ptr addrspace(1) %data) {
+; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_derived(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; PASS-CHECK-NEXT: [[ENTRY:.*]]:
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: br label %[[H:.*]]
+; PASS-CHECK: [[H]]:
+; PASS-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; PASS-CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[DATA]], i32 [[I]]
+; PASS-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
+; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
+; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
+; PASS-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; PASS-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; PASS-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; PASS-CHECK: [[EXIT]]:
+; PASS-CHECK-NEXT: [[EXT:%.*]] = zext i1 [[TMP0]] to i32
+; PASS-CHECK-NEXT: [[RL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[EXT]], i32 0)
+; PASS-CHECK-NEXT: [[OG:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; PASS-CHECK-NEXT: store i32 [[RL]], ptr addrspace(1) [[OG]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; COMB-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_derived(
+; COMB-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; COMB-CHECK-NEXT: [[ENTRY:.*]]:
+; COMB-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; COMB-CHECK-NEXT: br label %[[H:.*]]
+; COMB-CHECK: [[H]]:
+; COMB-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; COMB-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; COMB-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; COMB-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; COMB-CHECK: [[EXIT]]:
+; COMB-CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[I]] to i64
+; COMB-CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[DATA]], i64 [[TMP0]]
+; COMB-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
+; COMB-CHECK-NEXT: [[RL:%.*]] = call i1 @llvm.amdgcn.readlane.i1(i1 [[X]], i32 0)
+; COMB-CHECK-NEXT: [[TMP1:%.*]] = zext i1 [[RL]] to i32
+; COMB-CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[TID]] to i64
+; COMB-CHECK-NEXT: [[OG:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[OUT]], i64 [[TMP2]]
+; COMB-CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(1) [[OG]], align 4
+; COMB-CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %H
+
+H:
+ %i = phi i32 [ 0, %entry ], [ %inc, %H ]
+ %gep = getelementptr i32, ptr addrspace(1) %data, i32 %i
+ %v = load i32, ptr addrspace(1) %gep
+ %x = icmp sgt i32 %v, 0
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
+ %none = icmp eq i32 %ballot, 0
+ %inc = add i32 %i, 1
+ %div.exitx = icmp ugt i32 %inc, %tid
+ br i1 %div.exitx, label %exit, label %H ; divergent branch
+
+exit:
+ %ext = zext i1 %none to i32
+ %rl = call i32 @llvm.amdgcn.readlane.i32(i32 %ext, i32 0)
+ %og = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %rl, ptr addrspace(1) %og
+ ret void
+}
+
+define amdgpu_kernel void @temporal_divergence_readlane_of_not(ptr addrspace(1) %out, ptr addrspace(1) %data) {
+; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_of_not(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; PASS-CHECK-NEXT: [[ENTRY:.*]]:
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: br label %[[H:.*]]
+; PASS-CHECK: [[H]]:
+; PASS-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; PASS-CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[DATA]], i32 [[I]]
+; PASS-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
+; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; PASS-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; PASS-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; PASS-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; PASS-CHECK: [[EXIT]]:
+; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
+; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
+; PASS-CHECK-NEXT: [[RL:%.*]] = call i1 @llvm.amdgcn.readlane.i1(i1 [[TMP0]], i32 0)
+; PASS-CHECK-NEXT: [[EXT:%.*]] = zext i1 [[RL]] to i32
+; PASS-CHECK-NEXT: [[OG:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; PASS-CHECK-NEXT: store i32 [[EXT]], ptr addrspace(1) [[OG]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; COMB-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_of_not(
+; COMB-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
+; COMB-CHECK-NEXT: [[ENTRY:.*]]:
+; COMB-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; COMB-CHECK-NEXT: br label %[[H:.*]]
+; COMB-CHECK: [[H]]:
+; COMB-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
+; COMB-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
+; COMB-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
+; COMB-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
+; COMB-CHECK: [[EXIT]]:
+; COMB-CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[I]] to i64
+; COMB-CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[DATA]], i64 [[TMP0]]
+; COMB-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
+; COMB-CHECK-NEXT: [[RL:%.*]] = call i1 @llvm.amdgcn.readlane.i1(i1 [[X]], i32 0)
+; COMB-CHECK-NEXT: [[EXT:%.*]] = zext i1 [[RL]] to i32
+; COMB-CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[TID]] to i64
+; COMB-CHECK-NEXT: [[OG:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[OUT]], i64 [[TMP1]]
+; COMB-CHECK-NEXT: store i32 [[EXT]], ptr addrspace(1) [[OG]], align 4
+; COMB-CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %H
+
+H:
+ %i = phi i32 [ 0, %entry ], [ %inc, %H ]
+ %gep = getelementptr i32, ptr addrspace(1) %data, i32 %i
+ %v = load i32, ptr addrspace(1) %gep
+ %x = icmp sgt i32 %v, 0
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
+ %inc = add i32 %i, 1
+ %div.exitx = icmp ugt i32 %inc, %tid
+ br i1 %div.exitx, label %exit, label %H ; divergent branch
+
+exit:
+ %none = icmp eq i32 %ballot, 0
+ %rl = call i1 @llvm.amdgcn.readlane.i1(i1 %none, i32 0)
+ %ext = zext i1 %rl to i32
+ %og = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %ext, ptr addrspace(1) %og
+ ret void
+}
+
+define amdgpu_kernel void @ballot_chain_uniform(i32 %v, ptr addrspace(1) %out) {
+; PASS-CHECK-LABEL: define amdgpu_kernel void @ballot_chain_uniform(
+; PASS-CHECK-SAME: i32 [[V:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; PASS-CHECK-NEXT: [[ENTRY:.*:]]
+; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
+; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
+; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
+; PASS-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[TMP0]])
+; PASS-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; PASS-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; PASS-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; COMB-CHECK-LABEL: define amdgpu_kernel void @ballot_chain_uniform(
+; COMB-CHECK-SAME: i32 [[V:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; COMB-CHECK-NEXT: [[ENTRY:.*:]]
+; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
+; COMB-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
+; COMB-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
+; COMB-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; COMB-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
+; COMB-CHECK-NEXT: ret void
+;
+entry:
+ %x = icmp sgt i32 %v, 0
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
+ %none = icmp eq i32 %ballot, 0
+ %ballot2 = call i32 @llvm.amdgcn.ballot.i32(i1 %none)
+ %any = icmp ne i32 %ballot2, 0
+ %z = zext i1 %any to i32
+ store i32 %z, ptr addrspace(1) %out
+ ret void
+}
+
+declare i32 @llvm.amdgcn.ballot.i32(i1)
+declare i32 @llvm.amdgcn.readlane.i32(i32, i32)
+declare i1 @llvm.amdgcn.readlane.i1(i1, i32)
>From f52b8470ef5f3f8a1c1c4ad2aa1c6fe603b5638b Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 1 Sep 2026 12:49:28 +0200
Subject: [PATCH 2/2] simplify
---
.../AMDGPU/AMDGPUUniformIntrinsicCombine.cpp | 29 +++----
.../amdgpu-uniform-temporal-divergence.ll | 76 +------------------
2 files changed, 18 insertions(+), 87 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index b0e7c91c1dd26..1c5f6707f79f4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -41,16 +41,17 @@ using namespace llvm;
using namespace llvm::AMDGPU;
using namespace llvm::PatternMatch;
-/// Divergent per UI, or newly created (guards against address reuse).
+/// \p UI never analyzed the values this pass creates and reports them divergent.
+/// \p UniformUses lists the uses it proved uniform before rewriting them.
static bool isDivergentUse(const Use &U, const UniformityInfo &UI,
- const SmallPtrSetImpl<const Value *> &NewInsts) {
- return NewInsts.contains(U.get()) || UI.isDivergentAtUse(U);
+ const SmallPtrSetImpl<const Use *> &UniformUses) {
+ return !UniformUses.contains(&U) && UI.isDivergentAtUse(U);
}
/// Optimizes uniform intrinsics calls if their operand can be proven uniform.
-static bool optimizeUniformIntrinsic(IntrinsicInst &II,
- const UniformityInfo &UI,
- SmallPtrSetImpl<const Value *> &NewInsts) {
+static bool
+optimizeUniformIntrinsic(IntrinsicInst &II, const UniformityInfo &UI,
+ SmallPtrSetImpl<const Use *> &UniformUses) {
llvm::Intrinsic::ID IID = II.getIntrinsicID();
/// We deliberately do not simplify readfirstlane with a uniform argument, so
/// that frontends can use it to force a copy to SGPR and thereby prevent the
@@ -59,7 +60,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readlane: {
Value *Src = II.getArgOperand(0);
- if (isDivergentUse(II.getOperandUse(0), UI, NewInsts))
+ if (isDivergentUse(II.getOperandUse(0), UI, UniformUses))
return false;
LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');
II.replaceAllUsesWith(Src);
@@ -68,7 +69,7 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
}
case Intrinsic::amdgcn_ballot: {
Value *Src = II.getArgOperand(0);
- if (isDivergentUse(II.getOperandUse(0), UI, NewInsts))
+ if (isDivergentUse(II.getOperandUse(0), UI, UniformUses))
return false;
LLVM_DEBUG(dbgs() << "Found uniform ballot intrinsic: " << II << '\n');
@@ -84,7 +85,9 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
// Case: (icmp eq %ballot, 0) -> xor %ballot_arg, 1
Instruction *NotOp =
BinaryOperator::CreateNot(Src, "", ICmp->getIterator());
- NewInsts.insert(NotOp);
+ for (const Use &ICmpUse : ICmp->uses())
+ if (UI.isUniformAtUse(ICmpUse))
+ UniformUses.insert(&ICmpUse);
LLVM_DEBUG(dbgs() << "Replacing ICMP_EQ: " << *NotOp << '\n');
ICmp->replaceAllUsesWith(NotOp);
Changed = true;
@@ -107,14 +110,14 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
Use &Idx = II.getOperandUse(1);
// Like with readlane, if Value is uniform then just propagate it
- if (!isDivergentUse(Val, UI, NewInsts)) {
+ if (!isDivergentUse(Val, UI, UniformUses)) {
II.replaceAllUsesWith(Val);
II.eraseFromParent();
return true;
}
// Otherwise, when Index is uniform, this is just a readlane operation
- if (isDivergentUse(Idx, UI, NewInsts))
+ if (isDivergentUse(Idx, UI, UniformUses))
return false;
// The readlane intrinsic we want to call has the exact same function
@@ -133,13 +136,13 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
/// Iterates over intrinsic calls in the Function to optimize.
static bool runUniformIntrinsicCombine(Function &F, const UniformityInfo &UI) {
bool IsChanged = false;
- SmallPtrSet<const Value *, 8> NewInsts;
+ SmallPtrSet<const Use *, 8> UniformUses;
for (Instruction &I : make_early_inc_range(instructions(F))) {
auto *II = dyn_cast<IntrinsicInst>(&I);
if (!II)
continue;
- IsChanged |= optimizeUniformIntrinsic(*II, UI, NewInsts);
+ IsChanged |= optimizeUniformIntrinsic(*II, UI, UniformUses);
}
return IsChanged;
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
index 77d88e3ab6f66..21a8d5156eb69 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-temporal-divergence.ll
@@ -201,75 +201,6 @@ exit:
ret void
}
-define amdgpu_kernel void @temporal_divergence_readlane_derived(ptr addrspace(1) %out, ptr addrspace(1) %data) {
-; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_derived(
-; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
-; PASS-CHECK-NEXT: [[ENTRY:.*]]:
-; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; PASS-CHECK-NEXT: br label %[[H:.*]]
-; PASS-CHECK: [[H]]:
-; PASS-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
-; PASS-CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[DATA]], i32 [[I]]
-; PASS-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
-; PASS-CHECK-NEXT: [[X:%.*]] = icmp sgt i32 [[V]], 0
-; PASS-CHECK-NEXT: [[BALLOT:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
-; PASS-CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[X]], true
-; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
-; PASS-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
-; PASS-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
-; PASS-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
-; PASS-CHECK: [[EXIT]]:
-; PASS-CHECK-NEXT: [[EXT:%.*]] = zext i1 [[TMP0]] to i32
-; PASS-CHECK-NEXT: [[RL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[EXT]], i32 0)
-; PASS-CHECK-NEXT: [[OG:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
-; PASS-CHECK-NEXT: store i32 [[RL]], ptr addrspace(1) [[OG]], align 4
-; PASS-CHECK-NEXT: ret void
-;
-; COMB-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_derived(
-; COMB-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
-; COMB-CHECK-NEXT: [[ENTRY:.*]]:
-; COMB-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
-; COMB-CHECK-NEXT: br label %[[H:.*]]
-; COMB-CHECK: [[H]]:
-; COMB-CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[H]] ]
-; COMB-CHECK-NEXT: [[INC]] = add i32 [[I]], 1
-; COMB-CHECK-NEXT: [[DIV_EXITX:%.*]] = icmp ugt i32 [[INC]], [[TID]]
-; COMB-CHECK-NEXT: br i1 [[DIV_EXITX]], label %[[EXIT:.*]], label %[[H]]
-; COMB-CHECK: [[EXIT]]:
-; COMB-CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[I]] to i64
-; COMB-CHECK-NEXT: [[GEP:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[DATA]], i64 [[TMP0]]
-; COMB-CHECK-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
-; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
-; COMB-CHECK-NEXT: [[RL:%.*]] = call i1 @llvm.amdgcn.readlane.i1(i1 [[X]], i32 0)
-; COMB-CHECK-NEXT: [[TMP1:%.*]] = zext i1 [[RL]] to i32
-; COMB-CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[TID]] to i64
-; COMB-CHECK-NEXT: [[OG:%.*]] = getelementptr [4 x i8], ptr addrspace(1) [[OUT]], i64 [[TMP2]]
-; COMB-CHECK-NEXT: store i32 [[TMP1]], ptr addrspace(1) [[OG]], align 4
-; COMB-CHECK-NEXT: ret void
-;
-entry:
- %tid = call i32 @llvm.amdgcn.workitem.id.x()
- br label %H
-
-H:
- %i = phi i32 [ 0, %entry ], [ %inc, %H ]
- %gep = getelementptr i32, ptr addrspace(1) %data, i32 %i
- %v = load i32, ptr addrspace(1) %gep
- %x = icmp sgt i32 %v, 0
- %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %x)
- %none = icmp eq i32 %ballot, 0
- %inc = add i32 %i, 1
- %div.exitx = icmp ugt i32 %inc, %tid
- br i1 %div.exitx, label %exit, label %H ; divergent branch
-
-exit:
- %ext = zext i1 %none to i32
- %rl = call i32 @llvm.amdgcn.readlane.i32(i32 %ext, i32 0)
- %og = getelementptr i32, ptr addrspace(1) %out, i32 %tid
- store i32 %rl, ptr addrspace(1) %og
- ret void
-}
-
define amdgpu_kernel void @temporal_divergence_readlane_of_not(ptr addrspace(1) %out, ptr addrspace(1) %data) {
; PASS-CHECK-LABEL: define amdgpu_kernel void @temporal_divergence_readlane_of_not(
; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[DATA:%.*]]) {
@@ -349,7 +280,7 @@ define amdgpu_kernel void @ballot_chain_uniform(i32 %v, ptr addrspace(1) %out) {
; PASS-CHECK-NEXT: [[NONE:%.*]] = icmp eq i32 [[BALLOT]], 0
; PASS-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[TMP0]])
; PASS-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
-; PASS-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; PASS-CHECK-NEXT: [[Z:%.*]] = zext i1 [[TMP0]] to i32
; PASS-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
; PASS-CHECK-NEXT: ret void
;
@@ -357,9 +288,7 @@ define amdgpu_kernel void @ballot_chain_uniform(i32 %v, ptr addrspace(1) %out) {
; COMB-CHECK-SAME: i32 [[V:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
; COMB-CHECK-NEXT: [[ENTRY:.*:]]
; COMB-CHECK-NEXT: [[X:%.*]] = icmp slt i32 [[V]], 1
-; COMB-CHECK-NEXT: [[BALLOT2:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[X]])
-; COMB-CHECK-NEXT: [[ANY:%.*]] = icmp ne i32 [[BALLOT2]], 0
-; COMB-CHECK-NEXT: [[Z:%.*]] = zext i1 [[ANY]] to i32
+; COMB-CHECK-NEXT: [[Z:%.*]] = zext i1 [[X]] to i32
; COMB-CHECK-NEXT: store i32 [[Z]], ptr addrspace(1) [[OUT]], align 4
; COMB-CHECK-NEXT: ret void
;
@@ -375,5 +304,4 @@ entry:
}
declare i32 @llvm.amdgcn.ballot.i32(i1)
-declare i32 @llvm.amdgcn.readlane.i32(i32, i32)
declare i1 @llvm.amdgcn.readlane.i1(i1, i32)
More information about the llvm-commits
mailing list