[llvm] 62a9aad - [AMDGPU] Add optimization for llvm.amdgcn.wave.shuffle in uniform cases (#174795)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 14 09:49:46 PST 2026
Author: saxlungs
Date: 2026-01-14T17:49:42Z
New Revision: 62a9aadddc312c7782bdefe4dbe82be160efb977
URL: https://github.com/llvm/llvm-project/commit/62a9aadddc312c7782bdefe4dbe82be160efb977
DIFF: https://github.com/llvm/llvm-project/commit/62a9aadddc312c7782bdefe4dbe82be160efb977.diff
LOG: [AMDGPU] Add optimization for llvm.amdgcn.wave.shuffle in uniform cases (#174795)
When the llvm.amdgcn.wave.shuffle intrinsic is called with a uniform
Index operand, it is effectively the same as the llvm.amdgcn.readlane
intrinsic. This change handles this situation and replaces it with the
readlane intrinsic
---------
Signed-off-by: Domenic Nutile <domenic.nutile at gmail.com>
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
index c52eb4e477685..47d56adf18b7c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUniformIntrinsicCombine.cpp
@@ -107,6 +107,28 @@ static bool optimizeUniformIntrinsic(IntrinsicInst &II,
II.eraseFromParent();
return Changed;
}
+ case Intrinsic::amdgcn_wave_shuffle: {
+ Use &Val = II.getOperandUse(0);
+ Use &Idx = II.getOperandUse(1);
+
+ // Like with readlane, if Value is uniform then just propagate it
+ if (!isDivergentUseWithNew(Val, UI, Tracker)) {
+ II.replaceAllUsesWith(Val);
+ II.eraseFromParent();
+ return true;
+ }
+
+ // Otherwise, when Index is uniform, this is just a readlane operation
+ if (isDivergentUseWithNew(Idx, UI, Tracker))
+ return false;
+
+ // The readlane intrinsic we want to call has the exact same function
+ // signature, so we can quickly modify the instruction in-place
+ Module *Mod = II.getModule();
+ II.setCalledFunction(Intrinsic::getOrInsertDeclaration(
+ Mod, Intrinsic::amdgcn_readlane, II.getType()));
+ return true;
+ }
default:
return false;
}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine.ll
index 2fb7d8ab6af29..5391611742efd 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-uniform-intrinsic-combine.ll
@@ -805,3 +805,122 @@ define void @test_readlane_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src
call void asm sideeffect "; use $0", "s"(<8 x i16> %x)
ret void
}
+
+define amdgpu_kernel void @wave_shuffle_nonuniform_value_and_index(ptr addrspace(1) %out) {
+; CURRENT-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_and_index(
+; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CURRENT-CHECK-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CURRENT-CHECK-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[TID]], 1
+; CURRENT-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[TID]], i32 [[IDX]])
+; CURRENT-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; CURRENT-CHECK-NEXT: ret void
+;
+; PASS-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_and_index(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: [[IDX:%.*]] = add i32 [[TID]], 1
+; PASS-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[TID]], i32 [[IDX]])
+; PASS-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; DCE-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_and_index(
+; DCE-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; DCE-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; DCE-CHECK-NEXT: [[IDX:%.*]] = add i32 [[TID]], 1
+; DCE-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[TID]], i32 [[IDX]])
+; DCE-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; DCE-CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %tid2 = add i32 %tid, 1
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %tid, i32 %tid2)
+ store i32 %res, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @wave_shuffle_nonuniform_value_uniform_index(ptr addrspace(1) %in, ptr addrspace(1) %out) {
+; CURRENT-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_uniform_index(
+; CURRENT-CHECK-SAME: ptr addrspace(1) readnone captures(none) [[IN:%.*]], ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CURRENT-CHECK-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CURRENT-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[TID]], i32 5), !foo [[META0:![0-9]+]]
+; CURRENT-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; CURRENT-CHECK-NEXT: ret void
+;
+; PASS-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_uniform_index(
+; PASS-CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.readlane.i32(i32 [[TID]], i32 5), !foo [[META0:![0-9]+]]
+; PASS-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; DCE-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_nonuniform_value_uniform_index(
+; DCE-CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; DCE-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; DCE-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.readlane.i32(i32 [[TID]], i32 5), !foo [[META0:![0-9]+]]
+; DCE-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; DCE-CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %tid, i32 5), !foo !0
+ store i32 %res, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @wave_shuffle_uniform_value_nonuniform_index(ptr addrspace(1) %out) {
+; CURRENT-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_nonuniform_index(
+; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CURRENT-CHECK-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; CURRENT-CHECK-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[TID]], 1
+; CURRENT-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 42, i32 [[IDX]])
+; CURRENT-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; CURRENT-CHECK-NEXT: ret void
+;
+; PASS-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_nonuniform_index(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; PASS-CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; PASS-CHECK-NEXT: [[TID2:%.*]] = add i32 [[TID]], 1
+; PASS-CHECK-NEXT: store i32 42, ptr addrspace(1) [[OUT]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; DCE-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_nonuniform_index(
+; DCE-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; DCE-CHECK-NEXT: store i32 42, ptr addrspace(1) [[OUT]], align 4
+; DCE-CHECK-NEXT: ret void
+;
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %tid2 = add i32 %tid, 1
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 42, i32 %tid2)
+ store i32 %res, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @wave_shuffle_uniform_value_and_index(ptr addrspace(1) %out) {
+; CURRENT-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_and_index(
+; CURRENT-CHECK-SAME: ptr addrspace(1) writeonly captures(none) initializes((0, 4)) [[OUT:%.*]]) local_unnamed_addr #[[ATTR0]] {
+; CURRENT-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 42, i32 5)
+; CURRENT-CHECK-NEXT: store i32 [[RES]], ptr addrspace(1) [[OUT]], align 4
+; CURRENT-CHECK-NEXT: ret void
+;
+; PASS-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_and_index(
+; PASS-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; PASS-CHECK-NEXT: store i32 42, ptr addrspace(1) [[OUT]], align 4
+; PASS-CHECK-NEXT: ret void
+;
+; DCE-CHECK-LABEL: define amdgpu_kernel void @wave_shuffle_uniform_value_and_index(
+; DCE-CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; DCE-CHECK-NEXT: store i32 42, ptr addrspace(1) [[OUT]], align 4
+; DCE-CHECK-NEXT: ret void
+;
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 42, i32 5)
+ store i32 %res, ptr addrspace(1) %out
+ ret void
+}
+
+!0 = !{i32 5489}
+;.
+; CURRENT-CHECK: [[META0]] = !{i32 5489}
+;.
+; PASS-CHECK: [[META0]] = !{i32 5489}
+;.
+; DCE-CHECK: [[META0]] = !{i32 5489}
+;.
More information about the llvm-commits
mailing list