[llvm] [AMDGPU][GISel] Handle G_AMDGPU_COPY_VCC_SCC in isLaneMaskFromSameBlock (PR #202923)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 03:47:34 PDT 2026
https://github.com/anjenner created https://github.com/llvm/llvm-project/pull/202923
This avoids generating some redundant ANDs with exec.
>From 8dc1624754403f5ca9595d7e342718690ac543e7 Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 10 Jun 2026 06:21:12 -0400
Subject: [PATCH 1/2] [AMDGPU][GISel] Lower icmp i1 != 0 to ballot.
---
.../AMDGPU/AMDGPUInstructionSelector.cpp | 4 ++
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 22 ++++++++++
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 3 +-
.../atomic_optimizations_mul_one.ll | 24 +++--------
.../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 40 ++++---------------
.../GlobalISel/llvm.amdgcn.ballot.i32.ll | 3 +-
.../GlobalISel/llvm.amdgcn.ballot.i64.ll | 7 ++--
.../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 2 +-
8 files changed, 48 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 7fcbd5e207f41..de4b0855d0880 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1697,6 +1697,10 @@ static bool isLaneMaskFromSameBlock(Register Reg, MachineRegisterInfo &MRI,
return true;
}
+ // Lane mask generated by SCC to VCC copy (GlobalISel).
+ if (MI->getOpcode() == AMDGPU::G_AMDGPU_COPY_VCC_SCC)
+ return true;
+
// Lane mask generated using compare with same exec.
if (isa<GAnyCmp>(MI))
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 7269301b95f32..3911bc85e9aaa 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1449,6 +1449,28 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToNegMax(MI);
case AbsToS32:
return lowerAbsToS32(MI);
+ case IcmpI1ToBallot: {
+ // amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
+ // The src operand has been converted to VCC by the Vcc mapping.
+ assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
+
+ // Verify src1 is i1 false (constant 0).
+ Register Src1 = MI.getOperand(3).getReg();
+ auto Src1Const = getAnyConstantVRegValWithLookThrough(Src1, MRI);
+ if (!Src1Const || Src1Const->Value != 0)
+ return false;
+
+ // Verify predicate is ICMP_NE (33).
+ int64_t Pred = MI.getOperand(4).getImm();
+ if (Pred != CmpInst::ICMP_NE)
+ return false;
+
+ Register Dst = MI.getOperand(0).getReg();
+ Register Src = MI.getOperand(2).getReg();
+ B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
+ MI.eraseFromParent();
+ return true;
+ }
}
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index d9b1dd6289376..ec29b84727b2a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -338,7 +338,8 @@ enum LoweringMethodID {
InsVecEltToSel,
InsVecEltTo32,
AbsToNegMax,
- AbsToS32
+ AbsToS32,
+ IcmpI1ToBallot
};
enum FastRulesTypes {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index c5c1e8f1d9323..e2ecedff93716 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -36,7 +36,6 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
; GCN-LABEL: atomic_add:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -80,8 +79,7 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -134,7 +132,6 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
; GCN-LABEL: atomic_sub:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -178,8 +175,7 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -233,7 +229,6 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; GCN-LABEL: atomic_xor:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -280,8 +275,7 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -336,7 +330,6 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; GCN-LABEL: atomic_ptr_add:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -382,8 +375,7 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -438,7 +430,6 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; GCN-LABEL: atomic_ptr_sub:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -484,8 +475,7 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -541,7 +531,6 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; GCN-LABEL: atomic_ptr_xor:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -590,8 +579,7 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
+; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index 7690fb4d422ea..c2681726e52f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -1478,7 +1478,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1502,7 +1501,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1527,9 +1525,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB36_2
; GFX1250-NEXT: ; %bb.1:
@@ -1557,7 +1554,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1579,7 +1575,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1604,9 +1599,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB37_2
; GFX1250-NEXT: ; %bb.1:
@@ -1634,7 +1628,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1658,7 +1651,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1683,9 +1675,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB38_2
; GFX1250-NEXT: ; %bb.1:
@@ -1713,7 +1704,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1735,7 +1725,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1760,9 +1749,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB39_2
; GFX1250-NEXT: ; %bb.1:
@@ -1916,7 +1904,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1938,7 +1925,6 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -1963,9 +1949,8 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB43_2
; GFX1250-NEXT: ; %bb.1:
@@ -2307,7 +2292,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2328,7 +2312,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2351,9 +2334,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB51_2
; GFX1250-NEXT: ; %bb.1:
@@ -2376,7 +2358,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2397,7 +2378,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2420,9 +2400,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB52_2
; GFX1250-NEXT: ; %bb.1:
@@ -2445,7 +2424,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2466,7 +2444,6 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
@@ -2489,9 +2466,8 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1250-NEXT: s_cbranch_execz .LBB53_2
; GFX1250-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
index ad88b6030004b..91a6ca417d410 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll
@@ -21,7 +21,7 @@ define amdgpu_cs i32 @constant_false() {
define amdgpu_cs i32 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b32 s0, exec_lo, exec_lo
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
ret i32 %ballot
@@ -383,7 +383,6 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b32 s0, exec_lo, 0
-; CHECK-NEXT: s_and_b32 s0, s0, exec_lo
; CHECK-NEXT: s_cmp_le_i32 s0, 22
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
; CHECK-NEXT: ; %bb.1: ; %true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
index 9df5a811b85e6..a62355dbd2a78 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i64.ll
@@ -9,7 +9,8 @@ declare i64 @llvm.ctpop.i64(i64)
define amdgpu_cs i64 @constant_false() {
; CHECK-LABEL: constant_false:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b64 s[0:1], 0, exec
+; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: s_mov_b32 s1, 0
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 0)
ret i64 %ballot
@@ -20,7 +21,8 @@ define amdgpu_cs i64 @constant_false() {
define amdgpu_cs i64 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_and_b64 s[0:1], exec, exec
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
+; CHECK-NEXT: s_mov_b32 s1, exec_hi
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)
ret i64 %ballot
@@ -383,7 +385,6 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_cmp_lt_u32 s0, 12
; CHECK-NEXT: s_cselect_b64 s[0:1], exec, 0
-; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
; CHECK-NEXT: v_cmp_le_i64_e64 s[0:1], s[0:1], 22
; CHECK-NEXT: s_cmp_lg_u64 s[0:1], 0
; CHECK-NEXT: s_cbranch_scc1 .LBB18_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
index f3f41fac2be4f..2a688a7e4eeb6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll
@@ -32,7 +32,7 @@ define amdgpu_cs i64 @constant_true() {
;
; GISEL-LABEL: constant_true:
; GISEL: ; %bb.0:
-; GISEL-NEXT: s_and_b32 s0, exec_lo, exec_lo
+; GISEL-NEXT: s_mov_b32 s0, exec_lo
; GISEL-NEXT: s_mov_b32 s1, 0
; GISEL-NEXT: ; return to shader part epilog
%ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1)
>From 582519fb0defc4eacc42b981d865612bf375bdef Mon Sep 17 00:00:00 2001
From: Andrew Jenner <Andrew.Jenner at amd.com>
Date: Wed, 10 Jun 2026 06:33:53 -0400
Subject: [PATCH 2/2] Revert IcmpI1ToBallot parts of previous commit.
---
.../AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 22 -------------------
.../AMDGPU/AMDGPURegBankLegalizeRules.h | 3 +--
2 files changed, 1 insertion(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 3911bc85e9aaa..7269301b95f32 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -1449,28 +1449,6 @@ bool RegBankLegalizeHelper::lower(MachineInstr &MI,
return lowerAbsToNegMax(MI);
case AbsToS32:
return lowerAbsToS32(MI);
- case IcmpI1ToBallot: {
- // amdgcn.icmp(i1 src, i1 0, NE) -> ballot(src)
- // The src operand has been converted to VCC by the Vcc mapping.
- assert(cast<GIntrinsic>(&MI)->is(Intrinsic::amdgcn_icmp));
-
- // Verify src1 is i1 false (constant 0).
- Register Src1 = MI.getOperand(3).getReg();
- auto Src1Const = getAnyConstantVRegValWithLookThrough(Src1, MRI);
- if (!Src1Const || Src1Const->Value != 0)
- return false;
-
- // Verify predicate is ICMP_NE (33).
- int64_t Pred = MI.getOperand(4).getImm();
- if (Pred != CmpInst::ICMP_NE)
- return false;
-
- Register Dst = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(2).getReg();
- B.buildIntrinsic(Intrinsic::amdgcn_ballot, Dst).addUse(Src);
- MI.eraseFromParent();
- return true;
- }
}
return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index ec29b84727b2a..d9b1dd6289376 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -338,8 +338,7 @@ enum LoweringMethodID {
InsVecEltToSel,
InsVecEltTo32,
AbsToNegMax,
- AbsToS32,
- IcmpI1ToBallot
+ AbsToS32
};
enum FastRulesTypes {
More information about the llvm-commits
mailing list