[llvm] [AMDGPU] Produce ballot/icmp/fcmp lane masks at wavefront width (PR #201358)

Arseniy Obolenskiy via llvm-commits llvm-commits at lists.llvm.org
Wed Jun 3 08:12:37 PDT 2026


https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/201358

>From 3b49926b7c282066ac047edfdf5b6f0a77cf0fda Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 15:31:47 +0200
Subject: [PATCH 1/3] [AMDGPU] Read EXEC at wave-size width in icmp/fcmp
 instcombine fold

Reading read_register with the return type and a hardcoded "exec" name crashed isel on a width mismatch

Read EXEC at wave-size width with the matching register name, then zext/trunc to the actual return type
---
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp     | 16 +++--
 .../AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll  | 70 +++++++++++++++++++
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   |  6 +-
 3 files changed, 84 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 2370c379e75f5..8457b2594d3fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1644,15 +1644,19 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
         // intrinsic exposes) is one bit per thread, masked with the EXEC
         // register (which contains the bitmask of live threads). So a
         // comparison that always returns true is the same as a read of the
-        // EXEC register.
-        Metadata *MDArgs[] = {MDString::get(II.getContext(), "exec")};
+        // EXEC register. EXEC is wave-size bits wide, so read it at that width
+        // and zext/trunc to the return type.
+        Type *ExecTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
+        StringRef ExecName = ST->isWave32() ? "exec_lo" : "exec";
+        Metadata *MDArgs[] = {MDString::get(II.getContext(), ExecName)};
         MDNode *MD = MDNode::get(II.getContext(), MDArgs);
         Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
-        CallInst *NewCall = IC.Builder.CreateIntrinsic(Intrinsic::read_register,
-                                                       II.getType(), Args);
+        CallInst *NewCall =
+            IC.Builder.CreateIntrinsic(Intrinsic::read_register, ExecTy, Args);
         NewCall->addFnAttr(Attribute::Convergent);
-        NewCall->takeName(&II);
-        return IC.replaceInstUsesWith(II, NewCall);
+        Value *Result = IC.Builder.CreateZExtOrTrunc(NewCall, II.getType());
+        Result->takeName(&II);
+        return IC.replaceInstUsesWith(II, Result);
       }
 
       // Canonicalize constants to RHS.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
new file mode 100644
index 0000000000000..d9e3030e53c4d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE64 %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE32 %s
+
+; The "always true" amdgcn.icmp/amdgcn.fcmp fold reads EXEC, which is wave-size
+; bits wide. The read_register must use that width and register name regardless
+; of the return type, then zext/trunc.
+
+; Return type wider than the wave size on wave32: read i32 exec_lo, then zext.
+define i64 @fcmp_true_i64(float %x, float %y) {
+; WAVE64-LABEL: @fcmp_true_i64(
+; WAVE64-NEXT:    [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
+; WAVE64-NEXT:    ret i64 [[R]]
+;
+; WAVE32-LABEL: @fcmp_true_i64(
+; WAVE32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
+; WAVE32-NEXT:    [[R:%.*]] = zext i32 [[TMP1]] to i64
+; WAVE32-NEXT:    ret i64 [[R]]
+;
+  %r = call i64 @llvm.amdgcn.fcmp.i64.f32(float 0.0, float 0.0, i32 1)
+  ret i64 %r
+}
+
+define i64 @icmp_true_i64(i32 %x, i32 %y) {
+; WAVE64-LABEL: @icmp_true_i64(
+; WAVE64-NEXT:    [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
+; WAVE64-NEXT:    ret i64 [[R]]
+;
+; WAVE32-LABEL: @icmp_true_i64(
+; WAVE32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
+; WAVE32-NEXT:    [[R:%.*]] = zext i32 [[TMP1]] to i64
+; WAVE32-NEXT:    ret i64 [[R]]
+;
+  %r = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
+  ret i64 %r
+}
+
+; Return type narrower than the wave size on wave64: read i64 exec, then trunc.
+define i32 @fcmp_true_i32(float %x, float %y) {
+; WAVE64-LABEL: @fcmp_true_i32(
+; WAVE64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
+; WAVE64-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; WAVE64-NEXT:    ret i32 [[R]]
+;
+; WAVE32-LABEL: @fcmp_true_i32(
+; WAVE32-NEXT:    [[R:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
+; WAVE32-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @llvm.amdgcn.fcmp.i32.f32(float 0.0, float 0.0, i32 1)
+  ret i32 %r
+}
+
+declare i64 @llvm.amdgcn.fcmp.i64.f32(float, float, i32 immarg)
+declare i64 @llvm.amdgcn.icmp.i64.i32(i32, i32, i32 immarg)
+declare i32 @llvm.amdgcn.fcmp.i32.f32(float, float, i32 immarg)
+;.
+; WAVE64: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx900" }
+; WAVE64: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx900" }
+; WAVE64: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
+; WAVE64: attributes #[[ATTR3]] = { convergent }
+;.
+; WAVE32: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx1010" }
+; WAVE32: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx1010" }
+; WAVE32: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
+; WAVE32: attributes #[[ATTR3]] = { convergent }
+;.
+; WAVE64: [[META0]] = !{!"exec"}
+;.
+; WAVE32: [[META0]] = !{!"exec_lo"}
+;.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 87164cff4276e..bc43aaac6c567 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,7 +1882,8 @@ define i64 @icmp_constant_inputs_false() {
 
 define i64 @icmp_constant_inputs_true() {
 ; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2589,7 +2590,8 @@ define i64 @fcmp_constant_inputs_false() {
 
 define i64 @fcmp_constant_inputs_true() {
 ; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR21]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR21]]
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)

>From 434000bd4d1db0fe74b41eb7242e0bf795844607 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 16:14:19 +0200
Subject: [PATCH 2/3] Remove code

---
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp     | 19 +----
 .../AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll  | 70 -------------------
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   |  6 +-
 3 files changed, 3 insertions(+), 92 deletions(-)
 delete mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 8457b2594d3fc..cab324a78f48b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1639,24 +1639,7 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
           return IC.replaceInstUsesWith(
               II, IC.Builder.CreateSExt(CCmp, II.getType()));
         }
-
-        // The result of V_ICMP/V_FCMP assembly instructions (which this
-        // intrinsic exposes) is one bit per thread, masked with the EXEC
-        // register (which contains the bitmask of live threads). So a
-        // comparison that always returns true is the same as a read of the
-        // EXEC register. EXEC is wave-size bits wide, so read it at that width
-        // and zext/trunc to the return type.
-        Type *ExecTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
-        StringRef ExecName = ST->isWave32() ? "exec_lo" : "exec";
-        Metadata *MDArgs[] = {MDString::get(II.getContext(), ExecName)};
-        MDNode *MD = MDNode::get(II.getContext(), MDArgs);
-        Value *Args[] = {MetadataAsValue::get(II.getContext(), MD)};
-        CallInst *NewCall =
-            IC.Builder.CreateIntrinsic(Intrinsic::read_register, ExecTy, Args);
-        NewCall->addFnAttr(Attribute::Convergent);
-        Value *Result = IC.Builder.CreateZExtOrTrunc(NewCall, II.getType());
-        Result->takeName(&II);
-        return IC.replaceInstUsesWith(II, Result);
+        break;
       }
 
       // Canonicalize constants to RHS.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
deleted file mode 100644
index d9e3030e53c4d..0000000000000
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-cmp-exec-fold-wave-size.ll
+++ /dev/null
@@ -1,70 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE64 %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck -check-prefix=WAVE32 %s
-
-; The "always true" amdgcn.icmp/amdgcn.fcmp fold reads EXEC, which is wave-size
-; bits wide. The read_register must use that width and register name regardless
-; of the return type, then zext/trunc.
-
-; Return type wider than the wave size on wave32: read i32 exec_lo, then zext.
-define i64 @fcmp_true_i64(float %x, float %y) {
-; WAVE64-LABEL: @fcmp_true_i64(
-; WAVE64-NEXT:    [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
-; WAVE64-NEXT:    ret i64 [[R]]
-;
-; WAVE32-LABEL: @fcmp_true_i64(
-; WAVE32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR3:[0-9]+]]
-; WAVE32-NEXT:    [[R:%.*]] = zext i32 [[TMP1]] to i64
-; WAVE32-NEXT:    ret i64 [[R]]
-;
-  %r = call i64 @llvm.amdgcn.fcmp.i64.f32(float 0.0, float 0.0, i32 1)
-  ret i64 %r
-}
-
-define i64 @icmp_true_i64(i32 %x, i32 %y) {
-; WAVE64-LABEL: @icmp_true_i64(
-; WAVE64-NEXT:    [[R:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
-; WAVE64-NEXT:    ret i64 [[R]]
-;
-; WAVE32-LABEL: @icmp_true_i64(
-; WAVE32-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
-; WAVE32-NEXT:    [[R:%.*]] = zext i32 [[TMP1]] to i64
-; WAVE32-NEXT:    ret i64 [[R]]
-;
-  %r = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
-  ret i64 %r
-}
-
-; Return type narrower than the wave size on wave64: read i64 exec, then trunc.
-define i32 @fcmp_true_i32(float %x, float %y) {
-; WAVE64-LABEL: @fcmp_true_i32(
-; WAVE64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.read_register.i64(metadata [[META0]]) #[[ATTR3]]
-; WAVE64-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
-; WAVE64-NEXT:    ret i32 [[R]]
-;
-; WAVE32-LABEL: @fcmp_true_i32(
-; WAVE32-NEXT:    [[R:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR3]]
-; WAVE32-NEXT:    ret i32 [[R]]
-;
-  %r = call i32 @llvm.amdgcn.fcmp.i32.f32(float 0.0, float 0.0, i32 1)
-  ret i32 %r
-}
-
-declare i64 @llvm.amdgcn.fcmp.i64.f32(float, float, i32 immarg)
-declare i64 @llvm.amdgcn.icmp.i64.i32(i32, i32, i32 immarg)
-declare i32 @llvm.amdgcn.fcmp.i32.f32(float, float, i32 immarg)
-;.
-; WAVE64: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx900" }
-; WAVE64: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx900" }
-; WAVE64: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
-; WAVE64: attributes #[[ATTR3]] = { convergent }
-;.
-; WAVE32: attributes #[[ATTR0:[0-9]+]] = { "target-cpu"="gfx1010" }
-; WAVE32: attributes #[[ATTR1:[0-9]+]] = { convergent nocallback nofree nounwind willreturn memory(none) "target-cpu"="gfx1010" }
-; WAVE32: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(read) }
-; WAVE32: attributes #[[ATTR3]] = { convergent }
-;.
-; WAVE64: [[META0]] = !{!"exec"}
-;.
-; WAVE32: [[META0]] = !{!"exec_lo"}
-;.
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index bc43aaac6c567..4a323b09ce7b6 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,8 +1882,7 @@ define i64 @icmp_constant_inputs_false() {
 
 define i64 @icmp_constant_inputs_true() {
 ; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0:![0-9]+]]) #[[ATTR21:[0-9]+]]
-; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2590,8 +2589,7 @@ define i64 @fcmp_constant_inputs_false() {
 
 define i64 @fcmp_constant_inputs_true() {
 ; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.read_register.i32(metadata [[META0]]) #[[ATTR21]]
-; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.000000e+00, float 4.000000e+00, i32 4)
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)

>From 5bc38ed7f384232068e51488f0fe348e0185a0c9 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 3 Jun 2026 17:10:02 +0200
Subject: [PATCH 3/3] Address comments

---
 .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp     | 13 +++-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     | 26 +++----
 .../AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll   | 65 ++++++++++++++++++
 .../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll   | 67 ++++++++++++-------
 .../CodeGen/AMDGPU/nor-divergent-lanemask.ll  |  7 +-
 .../InstCombine/AMDGPU/amdgcn-intrinsics.ll   |  6 +-
 6 files changed, 140 insertions(+), 44 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index cab324a78f48b..64e12fb2c21da 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -1639,7 +1639,18 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
           return IC.replaceInstUsesWith(
               II, IC.Builder.CreateSExt(CCmp, II.getType()));
         }
-        break;
+
+        // The result of V_ICMP/V_FCMP assembly instructions (which this
+        // intrinsic exposes) is one bit per thread, masked with the EXEC
+        // register (which contains the bitmask of live threads). So a
+        // comparison that always returns true is the same as a read of the
+        // EXEC register. ballot(true) reads EXEC at the wave-size width, so
+        // zext/trunc the result to the intrinsic's return type.
+        Type *WaveTy = IC.Builder.getIntNTy(ST->getWavefrontSize());
+        Value *Ballot = IC.Builder.CreateIntrinsic(
+            Intrinsic::amdgcn_ballot, WaveTy, IC.Builder.getTrue());
+        Value *Result = IC.Builder.CreateZExtOrTrunc(Ballot, II.getType());
+        return IC.replaceInstUsesWith(II, Result);
       }
 
       // Canonicalize constants to RHS.
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 22e52c3c007f1..ca8f8784ba9fa 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7855,6 +7855,11 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
   SDValue Src = N->getOperand(1);
   SDLoc SL(N);
 
+  // AMDGPUISD::SETCC produces a wave-sized mask; emit it at that width and
+  // zext/trunc to the user-requested return type.
+  EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(),
+                                 TLI.getSubtarget()->getWavefrontSize());
+
   if (Src.getOpcode() == ISD::SETCC) {
     SDValue Op0 = Src.getOperand(0);
     SDValue Op1 = Src.getOperand(1);
@@ -7864,7 +7869,9 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
       Op1 = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, Op1);
     }
     // (ballot (ISD::SETCC ...)) -> (AMDGPUISD::SETCC ...)
-    return DAG.getNode(AMDGPUISD::SETCC, SL, VT, Op0, Op1, Src.getOperand(2));
+    SDValue SetCC =
+        DAG.getNode(AMDGPUISD::SETCC, SL, MaskVT, Op0, Op1, Src.getOperand(2));
+    return DAG.getZExtOrTrunc(SetCC, SL, VT);
   }
   if (const ConstantSDNode *Arg = dyn_cast<ConstantSDNode>(Src)) {
     // (ballot 0) -> 0
@@ -7873,23 +7880,18 @@ static SDValue lowerBALLOTIntrinsic(const SITargetLowering &TLI, SDNode *N,
 
     // (ballot 1) -> EXEC/EXEC_LO
     if (Arg->isOne()) {
-      Register Exec;
-      if (VT.getScalarSizeInBits() == 32)
-        Exec = AMDGPU::EXEC_LO;
-      else if (VT.getScalarSizeInBits() == 64)
-        Exec = AMDGPU::EXEC;
-      else
-        return SDValue();
-
-      return DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, VT);
+      Register Exec = MaskVT == MVT::i32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+      SDValue ExecVal = DAG.getCopyFromReg(DAG.getEntryNode(), SL, Exec, MaskVT);
+      return DAG.getZExtOrTrunc(ExecVal, SL, VT);
     }
   }
 
   // (ballot (i1 $src)) -> (AMDGPUISD::SETCC (i32 (zext $src)) (i32 0)
   // ISD::SETNE)
-  return DAG.getNode(
-      AMDGPUISD::SETCC, SL, VT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
+  SDValue SetCC = DAG.getNode(
+      AMDGPUISD::SETCC, SL, MaskVT, DAG.getZExtOrTrunc(Src, SL, MVT::i32),
       DAG.getConstant(0, SL, MVT::i32), DAG.getCondCode(ISD::SETNE));
+  return DAG.getZExtOrTrunc(SetCC, SL, VT);
 }
 
 static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll
new file mode 100644
index 0000000000000..618d91792bb28
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.wave64.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -global-isel=0 -mcpu=gfx900 < %s | FileCheck %s
+
+; ballot.i32 on a wave64 target: the i32 return type is narrower than the
+; wave mask. SelectionDAG lowering must emit the wave-sized SETCC and
+; truncate, otherwise ISel cannot select the wrong-width AMDGPUISD::SETCC.
+
+declare i32 @llvm.amdgcn.ballot.i32(i1)
+
+; Test ballot(0)
+
+define amdgpu_cs i32 @constant_false() {
+; CHECK-LABEL: constant_false:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    ; return to shader part epilog
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 0)
+  ret i32 %ballot
+}
+
+; Test ballot(1)
+
+define amdgpu_cs i32 @constant_true() {
+; CHECK-LABEL: constant_true:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_mov_b32 s0, exec_lo
+; CHECK-NEXT:    ; return to shader part epilog
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
+  ret i32 %ballot
+}
+
+; Test ballot of a non-comparison operation
+
+define amdgpu_cs i32 @non_compare(i32 %x) {
+; CHECK-LABEL: non_compare:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_and_b32_e32 v0, 1, v0
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v0
+; CHECK-NEXT:    ; return to shader part epilog
+  %trunc = trunc i32 %x to i1
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %trunc)
+  ret i32 %ballot
+}
+
+; Test ballot of comparisons
+
+define amdgpu_cs i32 @compare_ints(i32 %x, i32 %y) {
+; CHECK-LABEL: compare_ints:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_cmp_eq_u32_e64 s[0:1], v0, v1
+; CHECK-NEXT:    ; return to shader part epilog
+  %cmp = icmp eq i32 %x, %y
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+  ret i32 %ballot
+}
+
+define amdgpu_cs i32 @compare_floats(float %x, float %y) {
+; CHECK-LABEL: compare_floats:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_cmp_gt_f32_e64 s[0:1], v0, v1
+; CHECK-NEXT:    ; return to shader part epilog
+  %cmp = fcmp ogt float %x, %y
+  %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
+  ret i32 %ballot
+}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
index f3f41fac2be4f..a7f9fe019061d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
@@ -27,7 +27,7 @@ define amdgpu_cs i64 @constant_true() {
 ; DAGISEL-LABEL: constant_true:
 ; DAGISEL:       ; %bb.0:
 ; DAGISEL-NEXT:    s_mov_b32 s0, exec_lo
-; DAGISEL-NEXT:    s_mov_b32 s1, exec_hi
+; DAGISEL-NEXT:    s_mov_b32 s1, 0
 ; DAGISEL-NEXT:    ; return to shader part epilog
 ;
 ; GISEL-LABEL: constant_true:
@@ -95,12 +95,19 @@ define amdgpu_cs i64 @compare_floats(float %x, float %y) {
 }
 
 define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {
-; CHECK-LABEL: ctpop_of_ballot:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    v_cmp_gt_f32_e64 s0, v0, v1
-; CHECK-NEXT:    s_mov_b32 s1, 0
-; CHECK-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
-; CHECK-NEXT:    ; return to shader part epilog
+; DAGISEL-LABEL: ctpop_of_ballot:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v0, v1
+; DAGISEL-NEXT:    s_mov_b32 s1, 0
+; DAGISEL-NEXT:    s_bcnt1_i32_b32 s0, vcc_lo
+; DAGISEL-NEXT:    ; return to shader part epilog
+;
+; GISEL-LABEL: ctpop_of_ballot:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    v_cmp_gt_f32_e64 s0, v0, v1
+; GISEL-NEXT:    s_mov_b32 s1, 0
+; GISEL-NEXT:    s_bcnt1_i32_b64 s0, s[0:1]
+; GISEL-NEXT:    ; return to shader part epilog
   %cmp = fcmp ogt float %x, %y
   %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp)
   %bcnt = call i64 @llvm.ctpop.i64(i64 %ballot)
@@ -108,19 +115,31 @@ define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) {
 }
 
 define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) {
-; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    v_cmp_gt_u32_e64 s0, 12, v0
-; CHECK-NEXT:    s_mov_b32 s1, 0
-; CHECK-NEXT:    s_cmp_eq_u64 s[0:1], 0
-; CHECK-NEXT:    s_cbranch_scc1 .LBB7_2
-; CHECK-NEXT:  ; %bb.1: ; %true
-; CHECK-NEXT:    s_mov_b32 s0, 42
-; CHECK-NEXT:    s_branch .LBB7_3
-; CHECK-NEXT:  .LBB7_2: ; %false
-; CHECK-NEXT:    s_mov_b32 s0, 33
-; CHECK-NEXT:    s_branch .LBB7_3
-; CHECK-NEXT:  .LBB7_3:
+; DAGISEL-LABEL: branch_divergent_ballot64_ne_zero_compare:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 12, v0
+; DAGISEL-NEXT:    s_cbranch_vccz .LBB7_2
+; DAGISEL-NEXT:  ; %bb.1: ; %true
+; DAGISEL-NEXT:    s_mov_b32 s0, 42
+; DAGISEL-NEXT:    s_branch .LBB7_3
+; DAGISEL-NEXT:  .LBB7_2: ; %false
+; DAGISEL-NEXT:    s_mov_b32 s0, 33
+; DAGISEL-NEXT:    s_branch .LBB7_3
+; DAGISEL-NEXT:  .LBB7_3:
+;
+; GISEL-LABEL: branch_divergent_ballot64_ne_zero_compare:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    v_cmp_gt_u32_e64 s0, 12, v0
+; GISEL-NEXT:    s_mov_b32 s1, 0
+; GISEL-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GISEL-NEXT:    s_cbranch_scc1 .LBB7_2
+; GISEL-NEXT:  ; %bb.1: ; %true
+; GISEL-NEXT:    s_mov_b32 s0, 42
+; GISEL-NEXT:    s_branch .LBB7_3
+; GISEL-NEXT:  .LBB7_2: ; %false
+; GISEL-NEXT:    s_mov_b32 s0, 33
+; GISEL-NEXT:    s_branch .LBB7_3
+; GISEL-NEXT:  .LBB7_3:
   %c = icmp ult i32 %v, 12
   %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c)
   %ballot_ne_zero = icmp ne i64 %ballot, 0
@@ -136,12 +155,8 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) {
 ; DAGISEL:       ; %bb.0:
 ; DAGISEL-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 12, v0
 ; DAGISEL-NEXT:    v_cmp_lt_u32_e64 s0, 34, v1
-; DAGISEL-NEXT:    s_mov_b32 s1, 0
-; DAGISEL-NEXT:    s_and_b32 s0, vcc_lo, s0
-; DAGISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
-; DAGISEL-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
-; DAGISEL-NEXT:    s_cmp_eq_u64 s[0:1], 0
-; DAGISEL-NEXT:    s_cbranch_scc1 .LBB8_2
+; DAGISEL-NEXT:    s_and_b32 vcc_lo, vcc_lo, s0
+; DAGISEL-NEXT:    s_cbranch_vccz .LBB8_2
 ; DAGISEL-NEXT:  ; %bb.1: ; %true
 ; DAGISEL-NEXT:    s_mov_b32 s0, 42
 ; DAGISEL-NEXT:    s_branch .LBB8_3
diff --git a/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll b/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
index a180e0face33e..6d68081182677 100644
--- a/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
+++ b/llvm/test/CodeGen/AMDGPU/nor-divergent-lanemask.ll
@@ -78,15 +78,16 @@ define amdgpu_ps i64 @test_or_two_uses(i64 inreg %a, i64 inreg %b) {
 ; SDAG-W32-LABEL: test_or_two_uses:
 ; SDAG-W32:       ; %bb.0:
 ; SDAG-W32-NEXT:    s_or_b32 s0, s0, s2
-; SDAG-W32-NEXT:    s_mov_b32 s3, 0
+; SDAG-W32-NEXT:    s_mov_b32 s1, 0
 ; SDAG-W32-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
 ; SDAG-W32-NEXT:    s_xor_b32 s0, s0, -1
-; SDAG-W32-NEXT:    s_mov_b32 s1, s3
+; SDAG-W32-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; SDAG-W32-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
-; SDAG-W32-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; SDAG-W32-NEXT:    v_cmp_ne_u32_e64 s0, 0, v0
+; SDAG-W32-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; SDAG-W32-NEXT:    v_cmp_ne_u32_e64 s2, 0, v1
 ; SDAG-W32-NEXT:    s_and_b64 s[0:1], s[2:3], s[0:1]
+; SDAG-W32-NEXT:    s_mov_b32 s1, 0
 ; SDAG-W32-NEXT:    ; return to shader part epilog
 ;
 ; GISEL-W32-LABEL: test_or_two_uses:
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 4a323b09ce7b6..ed197ebde7dd8 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1882,7 +1882,8 @@ define i64 @icmp_constant_inputs_false() {
 
 define i64 @icmp_constant_inputs_true() {
 ; CHECK-LABEL: @icmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true)
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.icmp.i64.i32(i32 9, i32 8, i32 34)
@@ -2589,7 +2590,8 @@ define i64 @fcmp_constant_inputs_false() {
 
 define i64 @fcmp_constant_inputs_true() {
 ; CHECK-LABEL: @fcmp_constant_inputs_true(
-; CHECK-NEXT:    [[RESULT:%.*]] = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.000000e+00, float 4.000000e+00, i32 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true)
+; CHECK-NEXT:    [[RESULT:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-NEXT:    ret i64 [[RESULT]]
 ;
   %result = call i64 @llvm.amdgcn.fcmp.i64.f32(float 2.0, float 4.0, i32 4)



More information about the llvm-commits mailing list