[llvm-branch-commits] [llvm] cd02324 - [AMDGPU] Fold icmp (readfirstlane X), X when X is uniform (#234262)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 11 01:51:51 PST 2026
Author: Siu Chi Chan
Date: 2026-11-11T08:29:27+05:30
New Revision: cd02324364cb4510677ff3a1c1a5c0468312778d
URL: https://github.com/llvm/llvm-project/commit/cd02324364cb4510677ff3a1c1a5c0468312778d
DIFF: https://github.com/llvm/llvm-project/commit/cd02324364cb4510677ff3a1c1a5c0468312778d.diff
LOG: [AMDGPU] Fold icmp (readfirstlane X), X when X is uniform (#234262)
Added:
llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-cmp.ll
Modified:
llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
llvm/test/CodeGen/AMDGPU/amdgpu-simplify-uniform-waterfall.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index 08b419555d584..d0e1ca58d4bbe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -123,10 +123,40 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II, UniformityInfo &UI) {
return false;
}
-/// Iterates over intrinsic calls in the Function to optimize.
+/// Fold `icmp pred (readfirstlane X), X` when X is uniform at that use.
+static bool optimizeUniformCmpReadFirstLane(ICmpInst &Cmp, UniformityInfo &UI) {
+ Value *X;
+ IntrinsicInst *RFL;
+ if (!match(&Cmp,
+ m_c_ICmp(m_CombineAnd(m_Intrinsic<Intrinsic::amdgcn_readfirstlane>(
+ m_Value(X)),
+ m_AnyIntrinsic(RFL)),
+ m_Deferred(X))))
+ return false;
+
+ if (UI.isDivergentAtUse(RFL->getArgOperandUse(0)))
+ return false;
+
+ // If readfirstlane's operand is uniform then we know we are comparing two
+ // identical values.
+ bool Holds = ICmpInst::isTrueWhenEqual(Cmp.getPredicate());
+ LLVM_DEBUG(dbgs() << "Uniform readfirstlane operand, replacing " << Cmp
+ << " with " << (Holds ? "true" : "false") << '\n');
+ Cmp.replaceAllUsesWith(ConstantInt::get(Cmp.getType(), Holds));
+ UI.forgetValue(&Cmp);
+ Cmp.eraseFromParent();
+ return true;
+}
+
+/// Iterates over the Function, optimizing uniform intrinsic calls and
+/// comparisons against a readfirstlane of a uniform value.
static bool runUniformIntrinsicCombine(Function &F, UniformityInfo &UI) {
bool IsChanged = false;
for (Instruction &I : make_early_inc_range(instructions(F))) {
+ if (auto *Cmp = dyn_cast<ICmpInst>(&I)) {
+ IsChanged |= optimizeUniformCmpReadFirstLane(*Cmp, UI);
+ continue;
+ }
auto *II = dyn_cast<IntrinsicInst>(&I);
if (!II)
continue;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-uniform-waterfall.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-uniform-waterfall.ll
index 0fa013970ae2c..f4d9603eb797b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-uniform-waterfall.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-simplify-uniform-waterfall.ll
@@ -240,8 +240,7 @@ define protected amdgpu_kernel void @trivial_uniform_waterfall(ptr addrspace(1)
; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF:.*]]
; PASS-CHECK: [[IF]]:
; PASS-CHECK-NEXT: [[FIRST_ACTIVE_ID:%.*]] = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 0)
-; PASS-CHECK-NEXT: [[IS_FIRST_ACTIVE_ID:%.*]] = icmp eq i32 0, [[FIRST_ACTIVE_ID]]
-; PASS-CHECK-NEXT: br i1 [[IS_FIRST_ACTIVE_ID]], label %[[WORK:.*]], label %[[TAIL]]
+; PASS-CHECK-NEXT: br i1 true, label %[[WORK:.*]], label %[[TAIL]]
; PASS-CHECK: [[WORK]]:
; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4
; PASS-CHECK-NEXT: br label %[[TAIL]]
@@ -310,8 +309,7 @@ define protected amdgpu_kernel void @uniform_waterfall(ptr addrspace(1) %out, i3
; PASS-CHECK-NEXT: br i1 [[TMP0]], label %[[EXIT:.*]], label %[[IF:.*]]
; PASS-CHECK: [[IF]]:
; PASS-CHECK-NEXT: [[FIRST_ACTIVE_ID:%.*]] = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 [[MYMASK]])
-; PASS-CHECK-NEXT: [[IS_FIRST_ACTIVE_ID:%.*]] = icmp eq i32 [[MYMASK]], [[FIRST_ACTIVE_ID]]
-; PASS-CHECK-NEXT: br i1 [[IS_FIRST_ACTIVE_ID]], label %[[WORK:.*]], label %[[TAIL]]
+; PASS-CHECK-NEXT: br i1 true, label %[[WORK:.*]], label %[[TAIL]]
; PASS-CHECK: [[WORK]]:
; PASS-CHECK-NEXT: store i32 5, ptr addrspace(1) [[OUT]], align 4
; PASS-CHECK-NEXT: br label %[[TAIL]]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-cmp.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-cmp.ll
new file mode 100644
index 0000000000000..79107587c021e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine-cmp.ll
@@ -0,0 +1,236 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-uniform-intrinsic-combine -S < %s | FileCheck %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -O3 -S < %s | FileCheck %s --check-prefix=O3
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -amdgpu-enable-uniform-intrinsic-combine=0 -O3 -S < %s | FileCheck %s --check-prefix=COMBINE-OFF
+
+declare i32 @llvm.amdgcn.readfirstlane.i32(i32)
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare i32 @llvm.amdgcn.ballot.i32(i1)
+declare i1 @llvm.is.constant.i32(i32)
+declare i32 @llvm.amdgcn.permlane.down.i32(i32, i32, i32)
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
+declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
+
+; CHECK-LABEL: @all_predicates(
+; CHECK: store volatile i1 true
+; CHECK-NEXT: store volatile i1 false
+; CHECK-NEXT: store volatile i1 false
+; CHECK-NEXT: store volatile i1 true
+; CHECK-NEXT: store volatile i1 false
+; CHECK-NEXT: store volatile i1 true
+; CHECK-NEXT: store volatile i1 false
+; CHECK-NEXT: store volatile i1 true
+; CHECK-NEXT: store volatile i1 false
+; CHECK-NEXT: store volatile i1 true
+; CHECK-NOT: icmp
+define amdgpu_kernel void @all_predicates(ptr addrspace(1) %out, i32 %x) {
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %x)
+ %a = icmp eq i32 %rfl, %x
+ %b = icmp ne i32 %rfl, %x
+ %c = icmp sgt i32 %rfl, %x
+ %d = icmp sge i32 %rfl, %x
+ %e = icmp slt i32 %rfl, %x
+ %f = icmp sle i32 %rfl, %x
+ %g = icmp ugt i32 %rfl, %x
+ %h = icmp uge i32 %rfl, %x
+ %i = icmp ult i32 %rfl, %x
+ %j = icmp ule i32 %rfl, %x
+ store volatile i1 %a, ptr addrspace(1) %out
+ store volatile i1 %b, ptr addrspace(1) %out
+ store volatile i1 %c, ptr addrspace(1) %out
+ store volatile i1 %d, ptr addrspace(1) %out
+ store volatile i1 %e, ptr addrspace(1) %out
+ store volatile i1 %f, ptr addrspace(1) %out
+ store volatile i1 %g, ptr addrspace(1) %out
+ store volatile i1 %h, ptr addrspace(1) %out
+ store volatile i1 %i, ptr addrspace(1) %out
+ store volatile i1 %j, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @commuted_match(
+; CHECK: store i1 false
+; CHECK-NOT: icmp
+define amdgpu_kernel void @commuted_match(ptr addrspace(1) %out, i32 %x) {
+ %r1 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %x)
+ %r2 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %r1)
+ %cmp = icmp slt i32 %r1, %r2
+ store i1 %cmp, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @loop_phi_uniform_step(
+; CHECK: store i32
+; CHECK-NOT: icmp eq
+define amdgpu_kernel void @loop_phi_uniform_step(ptr addrspace(1) %out, i32 %step, i32 %n) {
+entry:
+ br label %loop
+loop:
+ %i = phi i32 [ 1, %entry ], [ %next, %loop ]
+ %acc = phi i32 [ 0, %entry ], [ %sum, %loop ]
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %i)
+ %cmp = icmp eq i32 %rfl, %i
+ %sel = select i1 %cmp, i32 %rfl, i32 0
+ %sum = add i32 %acc, %sel
+ %next = add i32 %i, %step
+ %done = icmp sge i32 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @readfirstlane_multi_use(
+; CHECK: call i32 @llvm.amdgcn.readfirstlane
+; CHECK: select i1 true
+; CHECK-NOT: icmp
+define amdgpu_kernel void @readfirstlane_multi_use(ptr addrspace(1) %out, i32 %x) {
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %x)
+ %cmp = icmp eq i32 %rfl, %x
+ %sel = select i1 %cmp, i32 %rfl, i32 0
+ store i32 %sel, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @split_uses_across_divergent_exit(
+; CHECK: store i1 true
+; CHECK-NOT: icmp eq
+define amdgpu_kernel void @split_uses_across_divergent_exit(ptr addrspace(1) %out) {
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %loop
+loop:
+ %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %i)
+ %inc = add i32 %i, 1
+ %done = icmp sgt i32 %inc, %tid
+ br i1 %done, label %exit, label %loop
+exit:
+ %cmp = icmp eq i32 %rfl, %i
+ store i1 %cmp, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @divergent_sources(
+; CHECK: icmp eq i32 %rfl1, %tid
+; CHECK: icmp eq i32 %rfl2, %add
+; CHECK: icmp slt i32 %rfl3, %tid
+define amdgpu_kernel void @divergent_sources(ptr addrspace(1) %out, i32 %x) {
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %rfl1 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %tid)
+ %a = icmp eq i32 %rfl1, %tid
+ %add = add i32 %x, %tid
+ %rfl2 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %add)
+ %b = icmp eq i32 %rfl2, %add
+ %rfl3 = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %tid)
+ %c = icmp slt i32 %rfl3, %tid
+ store volatile i1 %a, ptr addrspace(1) %out
+ store volatile i1 %b, ptr addrspace(1) %out
+ store volatile i1 %c, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @temporal_divergence(
+; CHECK: icmp eq
+define amdgpu_kernel void @temporal_divergence(ptr addrspace(1) %out) {
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %loop
+loop:
+ %i = phi i32 [ 0, %entry ], [ %inc, %loop ]
+ %inc = add i32 %i, 1
+ %done = icmp sgt i32 %inc, %tid
+ br i1 %done, label %exit, label %loop
+exit:
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %i)
+ %cmp = icmp eq i32 %rfl, %i
+ store i1 %cmp, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @
diff erent_ssa_value(
+; CHECK: icmp eq
+define amdgpu_kernel void @
diff erent_ssa_value(ptr addrspace(1) %out, i32 %x0, i32 %n) {
+entry:
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %x0)
+ br label %loop
+loop:
+ %x = phi i32 [ %x0, %entry ], [ %next, %loop ]
+ %cmp = icmp eq i32 %rfl, %x
+ %next = shl i32 %x, 1
+ %d = icmp sge i32 %next, %n
+ br i1 %d, label %exit, label %loop
+exit:
+ store i1 %cmp, ptr addrspace(1) %out
+ ret void
+}
+
+; O3-LABEL: @guarded_shuffle_loop(
+; O3-NOT: ds.bpermute
+; O3: call i32 @llvm.amdgcn.permlane.down.i32
+; COMBINE-OFF-LABEL: @guarded_shuffle_loop(
+; COMBINE-OFF-NOT: permlane.down
+; COMBINE-OFF: call i32 @llvm.amdgcn.ds.bpermute
+define amdgpu_kernel void @guarded_shuffle_loop(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %n) {
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %gep = getelementptr i32, ptr addrspace(1) %in, i32 %tid
+ %v = load i32, ptr addrspace(1) %gep
+ br label %loop
+loop:
+ %i = phi i32 [ 1, %entry ], [ %next, %latch ]
+ %acc = phi i32 [ %v, %entry ], [ %sum, %latch ]
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %i)
+ %ne = icmp ne i32 %rfl, %i
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %ne)
+ %isc = call i1 @llvm.is.constant.i32(i32 %ballot)
+ %z = icmp eq i32 %ballot, 0
+ %uniform = and i1 %isc, %z
+ br i1 %uniform, label %fast, label %slow
+fast:
+ %a = call i32 @llvm.amdgcn.permlane.down.i32(i32 %acc, i32 %rfl, i32 32)
+ br label %latch
+slow:
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = add i32 %lane, %i
+ %byte = shl i32 %idx, 2
+ %b = call i32 @llvm.amdgcn.ds.bpermute(i32 %byte, i32 %acc)
+ br label %latch
+latch:
+ %sel = phi i32 [ %a, %fast ], [ %b, %slow ]
+ %sum = add i32 %sel, %acc
+ %next = shl i32 %i, 1
+ %done = icmp sge i32 %next, %n
+ br i1 %done, label %exit, label %loop
+exit:
+ store i32 %sum, ptr addrspace(1) %out
+ ret void
+}
+
+; CHECK-LABEL: @uniform_waterfall(
+; CHECK: readfirstlane.i32(i32 %mymask)
+; CHECK-NEXT: br i1 true, label %{{.*}}, label %{{.*}}
+; O3-LABEL: @uniform_waterfall(
+; O3: store i32 5, ptr addrspace(1) %out
+; O3-NEXT: ret void
+define amdgpu_kernel void @uniform_waterfall(ptr addrspace(1) %out, i32 %mymask) {
+entry:
+ br label %while
+while:
+ %done = phi i1 [ false, %entry ], [ %new_done, %tail ]
+ %not_done = xor i1 %done, true
+ %ballot = tail call i32 @llvm.amdgcn.ballot.i32(i1 %not_done)
+ %is_done = icmp eq i32 %ballot, 0
+ br i1 %is_done, label %exit, label %if
+if:
+ %first_active_id = tail call noundef i32 @llvm.amdgcn.readfirstlane.i32(i32 %mymask)
+ %is_first_active_id = icmp eq i32 %mymask, %first_active_id
+ br i1 %is_first_active_id, label %work, label %tail
+work:
+ store i32 5, ptr addrspace(1) %out
+ br label %tail
+tail:
+ %new_done = phi i1 [ true, %work ], [ false, %if ]
+ br label %while
+exit:
+ ret void
+}
More information about the llvm-branch-commits
mailing list