[llvm] [AMDGPU] Produce ballot/icmp/fcmp lane masks at wavefront width (PR #201358)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 3 08:12:37 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/201358
>From 3b49926b7c282066ac047edfdf5b6f0a77cf0fda Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 15:31:47 +0200
Subject: [PATCH 1/3] [AMDGPU] Read EXEC at wave-size width in icmp/fcmp
instcombine fold
Reading read_register with the return type and a hardcoded "exec" name crashed isel on a width mismatch
Read EXEC at wave-size width with the matching register name, then zext/trunc to the actual return type
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 16 +++--
.../AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll | 70 +++++++++++++++++++
.../InstCombine/AMDGPU/amdgcn-intrinsics.ll | 6 +-
3 files changed, 84 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 2370c379e75f5..8457b2594d3fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1644,15 +1644,19 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
// intrinsic exposes) is one bit per thread, masked with the EXEC
// register (which contains the bitmask of live threads). So a
// comparison that always returns true is the same as a read of the
- // EXEC register.
- Metadata *MDArgs[] = {MDString::get(II.getContext(), "exec")};
+ // EXEC register. EXEC is wave-size bits wide, so read it at that width
+ // and zext/trunc to the return type.
+ Type *ExecTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
+ StringRef ExecName = ST->isWave32() ? "exec_lo" : "exec";
+ Metadata *MDArgs[] = {MDString::get(II.getContext(), ExecName)};
MDNode *MD = MDNode::get(II.getContext(), MDArgs);
Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
- CallInst *NewCall = IC.Builder.CreateIntrinsic(Intrinsic::read_register,
- II.getType(), Args);
+ CallInst *NewCall =
+ IC.Builder.CreateIntrinsic(Intrinsic::read_register, ExecTy, Args);
NewCall->addFnAttr(Attribute::Convergent);
- NewCall->takeName(&II);
- return IC.replaceInstUsesWith(II, NewCall);
+ Value *Result = IC.Builder.CreateZExtOrTrunc(NewCall, II.getType());
+ Result->takeName(&II);
+ return IC.replaceInstUsesWith(II, Result);
}
// Canonicalize constants to RHS.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
new file mode 100644
index 0000000000000..d9e3030e53c4d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE64 %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE32 %s
+
+; The "always true" amdgcn.icmp/amdgcn.fcmp fold reads EXEC, which is wave-size
+; bits wide. The read_register must use that width and register name regardless
+; of the return type, then zext/trunc.
+
+; Return type wider than the wave size on wave32: read i32 exec_lo, then zext.
+define i64 @fcmp_true_i64(float %x, float %y) {
+; WAVE64-LABEL: @fcmp_true_i64(
+; WAVE64-NEXT: [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
+; WAVE64-NEXT: ret i64 [[R]]
+;
+; WAVE32-LABEL: @fcmp_true_i64(
+; WAVE32-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
+; WAVE32-NEXT: [[R:%.*]] = zext i32 [[TMP1]] to i64
+; WAVE32-NEXT: ret i64 [[R]]
+;
+ %r = call i64 @llvm.amdgcn.fcmp.i64.f32(float 0.0, float 0.0, i32 1)
+ ret i64 %r
+}
+
+define i64 @icmp_true_i64(i32 %x, i32 %y) {
+; WAVE64-LABEL: @icmp_true_i64(
+; WAVE64-NEXT: [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
+; WAVE64-NEXT: ret i64 [[R]]
+;
+; WAVE32-LABEL: @icmp_true_i64(
+; WAVE32-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
+; WAVE32-NEXT: [[R:%.*]] = zext i32 [[TMP1]] to i64
+; WAVE32-NEXT: ret i64 [[R]]
+;
+ %r = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
+ ret i64 %r
+}
+
+; Return type narrower than the wave size on wave64: read i64 exec, then trunc.
+define i32 @fcmp_true_i32(float %x, float %y) {
+; WAVE64-LABEL: @fcmp_true_i32(
+; WAVE64-NEXT: [[TMP1:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
+; WAVE64-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; WAVE64-NEXT: ret i32 [[R]]
+;
+; WAVE32-LABEL: @fcmp_true_i32(
+; WAVE32-NEXT: [[R:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
+; WAVE32-NEXT: ret i32 [[R]]
+;
+ %r = call i32 @llvm.amdgcn.fcmp.i32.f32(float 0.0, float 0.0, i32 1)
+ ret i32 %r
+}
+
+declare i64 @llvm.amdgcn.fcmp.i64.f32(float, float, i32 immarg)
+declare i64 @llvm.amdgcn.icmp.i64.i32(i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.fcmp.i32.f32(float, float, i32 immarg)
+;.
+; WAVE64: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx900" }
+; WAVE64: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx900" }
+; WAVE64: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
+; WAVE64: attributes #[[ATTR3]] = { convergent }
+;.
+; WAVE32: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx1010" }
+; WAVE32: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx1010" }
+; WAVE32: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
+; WAVE32: attributes #[[ATTR3]] = { convergent }
+;.
+; WAVE64: [[META0]] = !{!"exec"}
+;.
+; WAVE32: [[META0]] = !{!"exec_lo"}
+;.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 87164cff4276e..bc43aaac6c567 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,7 +1882,8 @@ define i64 @icmp_constant_inputs_false() {
define i64 @icmp_constant_inputs_true() {
; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
+; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2589,7 +2590,8 @@ define i64 @fcmp_constant_inputs_false() {
define i64 @fcmp_constant_inputs_true() {
; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR21]]
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR21]]
+; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)
>From 434000bd4d1db0fe74b41eb7242e0bf795844607 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 16:14:19 +0200
Subject: [PATCH 2/3] Remove code
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 19 +----
.../AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll | 70 -------------------
.../InstCombine/AMDGPU/amdgcn-intrinsics.ll | 6 +-
3 files changed, 3 insertions(+), 92 deletions(-)
delete mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 8457b2594d3fc..cab324a78f48b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1639,24 +1639,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceInstUsesWith(
II, IC.Builder.CreateSExt(CCmp, II.getType()));
}
-
- // The result of V_ICMP/V_FCMP assembly instructions (which this
- // intrinsic exposes) is one bit per thread, masked with the EXEC
- // register (which contains the bitmask of live threads). So a
- // comparison that always returns true is the same as a read of the
- // EXEC register. EXEC is wave-size bits wide, so read it at that width
- // and zext/trunc to the return type.
- Type *ExecTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
- StringRef ExecName = ST->isWave32() ? "exec_lo" : "exec";
- Metadata *MDArgs[] = {MDString::get(II.getContext(), ExecName)};
- MDNode *MD = MDNode::get(II.getContext(), MDArgs);
- Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
- CallInst *NewCall =
- IC.Builder.CreateIntrinsic(Intrinsic::read_register, ExecTy, Args);
- NewCall->addFnAttr(Attribute::Convergent);
- Value *Result = IC.Builder.CreateZExtOrTrunc(NewCall, II.getType());
- Result->takeName(&II);
- return IC.replaceInstUsesWith(II, Result);
+ break;
}
// Canonicalize constants to RHS.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
deleted file mode 100644
index d9e3030e53c4d..0000000000000
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
+++ /dev/null
@@ -1,70 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE64 %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE32 %s
-
-; The "always true" amdgcn.icmp/amdgcn.fcmp fold reads EXEC, which is wave-size
-; bits wide. The read_register must use that width and register name regardless
-; of the return type, then zext/trunc.
-
-; Return type wider than the wave size on wave32: read i32 exec_lo, then zext.
-define i64 @fcmp_true_i64(float %x, float %y) {
-; WAVE64-LABEL: @fcmp_true_i64(
-; WAVE64-NEXT: [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
-; WAVE64-NEXT: ret i64 [[R]]
-;
-; WAVE32-LABEL: @fcmp_true_i64(
-; WAVE32-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
-; WAVE32-NEXT: [[R:%.*]] = zext i32 [[TMP1]] to i64
-; WAVE32-NEXT: ret i64 [[R]]
-;
- %r = call i64 @llvm.amdgcn.fcmp.i64.f32(float 0.0, float 0.0, i32 1)
- ret i64 %r
-}
-
-define i64 @icmp_true_i64(i32 %x, i32 %y) {
-; WAVE64-LABEL: @icmp_true_i64(
-; WAVE64-NEXT: [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
-; WAVE64-NEXT: ret i64 [[R]]
-;
-; WAVE32-LABEL: @icmp_true_i64(
-; WAVE32-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
-; WAVE32-NEXT: [[R:%.*]] = zext i32 [[TMP1]] to i64
-; WAVE32-NEXT: ret i64 [[R]]
-;
- %r = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
- ret i64 %r
-}
-
-; Return type narrower than the wave size on wave64: read i64 exec, then trunc.
-define i32 @fcmp_true_i32(float %x, float %y) {
-; WAVE64-LABEL: @fcmp_true_i32(
-; WAVE64-NEXT: [[TMP1:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
-; WAVE64-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i32
-; WAVE64-NEXT: ret i32 [[R]]
-;
-; WAVE32-LABEL: @fcmp_true_i32(
-; WAVE32-NEXT: [[R:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
-; WAVE32-NEXT: ret i32 [[R]]
-;
- %r = call i32 @llvm.amdgcn.fcmp.i32.f32(float 0.0, float 0.0, i32 1)
- ret i32 %r
-}
-
-declare i64 @llvm.amdgcn.fcmp.i64.f32(float, float, i32 immarg)
-declare i64 @llvm.amdgcn.icmp.i64.i32(i32, i32, i32 immarg)
-declare i32 @llvm.amdgcn.fcmp.i32.f32(float, float, i32 immarg)
-;.
-; WAVE64: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx900" }
-; WAVE64: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx900" }
-; WAVE64: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
-; WAVE64: attributes #[[ATTR3]] = { convergent }
-;.
-; WAVE32: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx1010" }
-; WAVE32: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx1010" }
-; WAVE32: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
-; WAVE32: attributes #[[ATTR3]] = { convergent }
-;.
-; WAVE64: [[META0]] = !{!"exec"}
-;.
-; WAVE32: [[META0]] = !{!"exec_lo"}
-;.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index bc43aaac6c567..4a323b09ce7b6 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,8 +1882,7 @@ define i64 @icmp_constant_inputs_false() {
define i64 @icmp_constant_inputs_true() {
; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
-; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2590,8 +2589,7 @@ define i64 @fcmp_constant_inputs_false() {
define i64 @fcmp_constant_inputs_true() {
; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR21]]
-; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.000000e+00, float 4.000000e+00, i32 4)
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)
>From 5bc38ed7f384232068e51488f0fe348e0185a0c9 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 17:10:02 +0200
Subject: [PATCH 3/3] Address comments
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 13 +++-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 26 +++----
.../AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll | 65 ++++++++++++++++++
.../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 67 ++++++++++++-------
.../CodeGen/AMDGPU/nor-divergent-lanemask.ll | 7 +-
.../InstCombine/AMDGPU/amdgcn-intrinsics.ll | 6 +-
6 files changed, 140 insertions(+), 44 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index cab324a78f48b..64e12fb2c21da 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1639,7 +1639,18 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceInstUsesWith(
II, IC.Builder.CreateSExt(CCmp, II.getType()));
}
- break;
+
+ // The result of V_ICMP/V_FCMP assembly instructions (which this
+ // intrinsic exposes) is one bit per thread, masked with the EXEC
+ // register (which contains the bitmask of live threads). So a
+ // comparison that always returns true is the same as a read of the
+ // EXEC register. ballot(true) reads EXEC at the wave-size width, so
+ // zext/trunc the result to the intrinsic's return type.
+ Type *WaveTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
+ Value *Ballot = IC.Builder.CreateIntrinsic(
+ Intrinsic::amdgcn_ballot, WaveTy, IC.Builder.getTrue());
+ Value *Result = IC.Builder.CreateZExtOrTrunc(Ballot, II.getType());
+ return IC.replaceInstUsesWith(II, Result);
}
// Canonicalize constants to RHS.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 22e52c3c007f1..ca8f8784ba9fa 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7855,6 +7855,11 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
SDValue Src = N->getOperand(1);
SDLoc SL(N);
+ // AMDGPUISD::SETCC produces a wave-sized mask; emit it at that width and
+ // zext/trunc to the user-requested return type.
+ EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(),
+ TLI.getSubtarget()->getWavefrontSize());
+
if (Src.getOpcode() == ISD::SETCC) {
SDValue Op0 = Src.getOperand(0);
SDValue Op1 = Src.getOperand(1);
@@ -7864,7 +7869,9 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
Op1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op1);
}
// (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
- return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Op0, Op1, Src.getOperand(2));
+ SDValue SetCC =
+ DAG.getNode(AMDGPUISD::SETCC, SL, MaskVT, Op0, Op1, Src.getOperand(2));
+ return DAG.getZExtOrTrunc(SetCC, SL, VT);
}
if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
// (ballot 0) -> 0
@@ -7873,23 +7880,18 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
// (ballot 1) -> EXEC/EXEC_LO
if (Arg->isOne()) {
- Register Exec;
- if (VT.getScalarSizeInBits() == 32)
- Exec = AMDGPU::EXEC_LO;
- else if (VT.getScalarSizeInBits() == 64)
- Exec = AMDGPU::EXEC;
- else
- return SDValue();
-
- return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
+ Register Exec = MaskVT == MVT::i32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+ SDValue ExecVal = DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, MaskVT);
+ return DAG.getZExtOrTrunc(ExecVal, SL, VT);
}
}
// (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
// ISD::SETNE)
- return DAG.getNode(
- AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
+ SDValue SetCC = DAG.getNode(
+ AMDGPUISD::SETCC, SL, MaskVT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
+ return DAG.getZExtOrTrunc(SetCC, SL, VT);
}
static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll
new file mode 100644
index 0000000000000..618d91792bb28
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx900 < %s | FileCheck %s
+
+; ballot.i32 on a wave64 target: the i32 return type is narrower than the
+; wave mask. SelectionDAG lowering must emit the wave-sized SETCC and
+; truncate, otherwise ISel cannot select the wrong-width AMDGPUISD::SETCC.
+
+declare i32 @llvm.amdgcn.ballot.i32(i1)
+
+; Test ballot(0)
+
+define amdgpu_cs i32 @constant_false() {
+; CHECK-LABEL: constant_false:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: ; return to shader part epilog
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 0)
+ ret i32 %ballot
+}
+
+; Test ballot(1)
+
+define amdgpu_cs i32 @constant_true() {
+; CHECK-LABEL: constant_true:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
+; CHECK-NEXT: ; return to shader part epilog
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
+ ret i32 %ballot
+}
+
+; Test ballot of a non-comparison operation
+
+define amdgpu_cs i32 @non_compare(i32 %x) {
+; CHECK-LABEL: non_compare:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v0
+; CHECK-NEXT: ; return to shader part epilog
+ %trunc = trunc i32 %x to i1
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %trunc)
+ ret i32 %ballot
+}
+
+; Test ballot of comparisons
+
+define amdgpu_cs i32 @compare_ints(i32 %x, i32 %y) {
+; CHECK-LABEL: compare_ints:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[0:1], v0, v1
+; CHECK-NEXT: ; return to shader part epilog
+ %cmp = icmp eq i32 %x, %y
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+ ret i32 %ballot
+}
+
+define amdgpu_cs i32 @compare_floats(float %x, float %y) {
+; CHECK-LABEL: compare_floats:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: v_cmp_gt_f32_e64 s[0:1], v0, v1
+; CHECK-NEXT: ; return to shader part epilog
+ %cmp = fcmp ogt float %x, %y
+ %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+ ret i32 %ballot
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
index f3f41fac2be4f..a7f9fe019061d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
@@ -27,7 +27,7 @@ define amdgpu_cs i64 @constant_true() {
; DAGISEL-LABEL: constant_true:
; DAGISEL: ; %bb.0:
; DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; DAGISEL-NEXT: s_mov_b32 s1, exec_hi
+; DAGISEL-NEXT: s_mov_b32 s1, 0
; DAGISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: constant_true:
@@ -95,12 +95,19 @@ define amdgpu_cs i64 @compare_floats(float %x, float %y) {
}
define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {
-; CHECK-LABEL: ctpop_of_ballot:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: v_cmp_gt_f32_e64 s0, v0, v1
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; CHECK-NEXT: ; return to shader part epilog
+; DAGISEL-LABEL: ctpop_of_ballot:
+; DAGISEL: ; %bb.0:
+; DAGISEL-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1
+; DAGISEL-NEXT: s_mov_b32 s1, 0
+; DAGISEL-NEXT: s_bcnt1_i32_b32 s0, vcc_lo
+; DAGISEL-NEXT: ; return to shader part epilog
+;
+; GISEL-LABEL: ctpop_of_ballot:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: v_cmp_gt_f32_e64 s0, v0, v1
+; GISEL-NEXT: s_mov_b32 s1, 0
+; GISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GISEL-NEXT: ; return to shader part epilog
%cmp = fcmp ogt float %x, %y
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)
%bcnt = call i64 @llvm.ctpop.i64(i64 %ballot)
@@ -108,19 +115,31 @@ define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {
}
define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) {
-; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0
-; CHECK-NEXT: s_mov_b32 s1, 0
-; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0
-; CHECK-NEXT: s_cbranch_scc1 .LBB7_2
-; CHECK-NEXT: ; %bb.1: ; %true
-; CHECK-NEXT: s_mov_b32 s0, 42
-; CHECK-NEXT: s_branch .LBB7_3
-; CHECK-NEXT: .LBB7_2: ; %false
-; CHECK-NEXT: s_mov_b32 s0, 33
-; CHECK-NEXT: s_branch .LBB7_3
-; CHECK-NEXT: .LBB7_3:
+; DAGISEL-LABEL: branch_divergent_ballot64_ne_zero_compare:
+; DAGISEL: ; %bb.0:
+; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0
+; DAGISEL-NEXT: s_cbranch_vccz .LBB7_2
+; DAGISEL-NEXT: ; %bb.1: ; %true
+; DAGISEL-NEXT: s_mov_b32 s0, 42
+; DAGISEL-NEXT: s_branch .LBB7_3
+; DAGISEL-NEXT: .LBB7_2: ; %false
+; DAGISEL-NEXT: s_mov_b32 s0, 33
+; DAGISEL-NEXT: s_branch .LBB7_3
+; DAGISEL-NEXT: .LBB7_3:
+;
+; GISEL-LABEL: branch_divergent_ballot64_ne_zero_compare:
+; GISEL: ; %bb.0:
+; GISEL-NEXT: v_cmp_gt_u32_e64 s0, 12, v0
+; GISEL-NEXT: s_mov_b32 s1, 0
+; GISEL-NEXT: s_cmp_eq_u64 s[0:1], 0
+; GISEL-NEXT: s_cbranch_scc1 .LBB7_2
+; GISEL-NEXT: ; %bb.1: ; %true
+; GISEL-NEXT: s_mov_b32 s0, 42
+; GISEL-NEXT: s_branch .LBB7_3
+; GISEL-NEXT: .LBB7_2: ; %false
+; GISEL-NEXT: s_mov_b32 s0, 33
+; GISEL-NEXT: s_branch .LBB7_3
+; GISEL-NEXT: .LBB7_3:
%c = icmp ult i32 %v, 12
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)
%ballot_ne_zero = icmp ne i64 %ballot, 0
@@ -136,12 +155,8 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) {
; DAGISEL: ; %bb.0:
; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0
; DAGISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1
-; DAGISEL-NEXT: s_mov_b32 s1, 0
-; DAGISEL-NEXT: s_and_b32 s0, vcc_lo, s0
-; DAGISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; DAGISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0
-; DAGISEL-NEXT: s_cmp_eq_u64 s[0:1], 0
-; DAGISEL-NEXT: s_cbranch_scc1 .LBB8_2
+; DAGISEL-NEXT: s_and_b32 vcc_lo, vcc_lo, s0
+; DAGISEL-NEXT: s_cbranch_vccz .LBB8_2
; DAGISEL-NEXT: ; %bb.1: ; %true
; DAGISEL-NEXT: s_mov_b32 s0, 42
; DAGISEL-NEXT: s_branch .LBB8_3
diff --git a/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll b/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
index a180e0face33e..6d68081182677 100644
--- a/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
+++ b/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
@@ -78,15 +78,16 @@ define amdgpu_ps i64 @test_or_two_uses(i64 inreg %a, i64 inreg %b) {
; SDAG-W32-LABEL: test_or_two_uses:
; SDAG-W32: ; %bb.0:
; SDAG-W32-NEXT: s_or_b32 s0, s0, s2
-; SDAG-W32-NEXT: s_mov_b32 s3, 0
+; SDAG-W32-NEXT: s_mov_b32 s1, 0
; SDAG-W32-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; SDAG-W32-NEXT: s_xor_b32 s0, s0, -1
-; SDAG-W32-NEXT: s_mov_b32 s1, s3
+; SDAG-W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; SDAG-W32-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; SDAG-W32-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; SDAG-W32-NEXT: v_cmp_ne_u32_e64 s0, 0, v0
+; SDAG-W32-NEXT: s_delay_alu instid0(VALU_DEP_2)
; SDAG-W32-NEXT: v_cmp_ne_u32_e64 s2, 0, v1
; SDAG-W32-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; SDAG-W32-NEXT: s_mov_b32 s1, 0
; SDAG-W32-NEXT: ; return to shader part epilog
;
; GISEL-W32-LABEL: test_or_two_uses:
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 4a323b09ce7b6..ed197ebde7dd8 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,7 +1882,8 @@ define i64 @icmp_constant_inputs_false() {
define i64 @icmp_constant_inputs_true() {
; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true)
+; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2589,7 +2590,8 @@ define i64 @fcmp_constant_inputs_false() {
define i64 @fcmp_constant_inputs_true() {
; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.000000e+00, float 4.000000e+00, i32 4)
+; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true)
+; CHECK-NEXT: [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
; CHECK-NEXT: ret i64 [[RESULT]]
;
%result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)
More information about the llvm-commits
mailing list