[llvm] 868e3b8 - [AMDGPU] Fix set_inactive known bits to intersect both operands (#201817)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 07:00:22 PDT 2026
Author: Arseniy Obolenskiy
Date: 2026-07-15T16:00:17+02:00
New Revision: 868e3b898dcf243f97885ce0d857a2827f4d1fa3
URL: https://github.com/llvm/llvm-project/commit/868e3b898dcf243f97885ce0d857a2827f4d1fa3
DIFF: https://github.com/llvm/llvm-project/commit/868e3b898dcf243f97885ce0d857a2827f4d1fa3.diff
LOG: [AMDGPU] Fix set_inactive known bits to intersect both operands (#201817)
SimplifyDemandedBitsForTargetNode grouped amdgcn_set_inactive with the
single-source readfirstlane/readlane/wwm, taking known bits from operand
1 only
Handle the data from the operand 2 inactive lanes as well
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.chain.arg.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index ccf73a6aeef84..855bc18347b3c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5855,13 +5855,28 @@ bool AMDGPUTargetLowering::SimplifyDemandedBitsForTargetNode(
switch (Op.getConstantOperandVal(0)) {
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_readlane:
- case Intrinsic::amdgcn_set_inactive:
case Intrinsic::amdgcn_wwm: {
if (SimplifyDemandedBits(Op.getOperand(1), OriginalDemandedBits,
OriginalDemandedElts, Known, TLO, Depth + 1))
return true;
break;
}
+ case Intrinsic::amdgcn_set_inactive:
+ case Intrinsic::amdgcn_set_inactive_chain_arg: {
+ // The result is operand 1 in active lanes and operand 2 in inactive
+ // lanes, so the known bits are the intersection of both operands.
+ KnownBits KnownValue, KnownInactive;
+ if (SimplifyDemandedBits(Op.getOperand(1), OriginalDemandedBits,
+ OriginalDemandedElts, KnownValue, TLO,
+ Depth + 1))
+ return true;
+ if (SimplifyDemandedBits(Op.getOperand(2), OriginalDemandedBits,
+ OriginalDemandedElts, KnownInactive, TLO,
+ Depth + 1))
+ return true;
+ Known = KnownValue.intersectWith(KnownInactive);
+ break;
+ }
default:
break;
}
diff --git a/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
index f6cf0a59b8e70..b330c79eb5d7b 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
@@ -6,24 +6,27 @@
define amdgpu_hs void @wwm(i32 inreg %arg, ptr addrspace(8) inreg %buffer) {
; GCN-LABEL: wwm:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_mov_b32 s7, s4
-; GCN-NEXT: s_mov_b32 s4, s1
-; GCN-NEXT: s_mov_b32 s1, 16
; GCN-NEXT: s_mov_b32 s6, s3
; GCN-NEXT: s_mov_b32 s5, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: s_mov_b32 s7, s4
+; GCN-NEXT: s_mov_b32 s4, s1
+; GCN-NEXT: s_mov_b32 s1, 1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 1, 4, s[2:3]
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: s_cmp_lg_u32 s0, 0
-; GCN-NEXT: s_mov_b32 s0, 0
-; GCN-NEXT: s_cbranch_scc1 .LBB0_2
-; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mov_b32 s0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: s_cbranch_scc0 .LBB0_2
+; GCN-NEXT: ; %bb.1: ; %bb42
+; GCN-NEXT: s_mov_b32 s1, 0
; GCN-NEXT: .LBB0_2: ; %bb602
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s1, v1
; GCN-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_4
; GCN-NEXT: ; %bb.3: ; %bb49
-; GCN-NEXT: v_mov_b32_e32 v0, 1.0
-; GCN-NEXT: tbuffer_store_format_x v0, off, s[4:7], 1 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offset:4 glc
+; GCN-NEXT: v_mov_b32_e32 v1, 1.0
+; GCN-NEXT: tbuffer_store_format_x v1, off, s[4:7], 1 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offset:4 glc
; GCN-NEXT: .LBB0_4: ; %UnifiedReturnBlock
; GCN-NEXT: s_endpgm
entry:
@@ -58,24 +61,27 @@ work:
define amdgpu_hs void @strict_wwm(i32 inreg %arg, ptr addrspace(8) inreg %buffer) {
; GCN-LABEL: strict_wwm:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_mov_b32 s7, s4
-; GCN-NEXT: s_mov_b32 s4, s1
-; GCN-NEXT: s_mov_b32 s1, 16
; GCN-NEXT: s_mov_b32 s6, s3
; GCN-NEXT: s_mov_b32 s5, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: s_mov_b32 s7, s4
+; GCN-NEXT: s_mov_b32 s4, s1
+; GCN-NEXT: s_mov_b32 s1, 1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 1, 4, s[2:3]
+; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: s_cmp_lg_u32 s0, 0
-; GCN-NEXT: s_mov_b32 s0, 0
-; GCN-NEXT: s_cbranch_scc1 .LBB1_2
-; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_mov_b32 s0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: s_cbranch_scc0 .LBB1_2
+; GCN-NEXT: ; %bb.1: ; %bb42
+; GCN-NEXT: s_mov_b32 s1, 0
; GCN-NEXT: .LBB1_2: ; %bb602
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s1, v1
; GCN-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_4
; GCN-NEXT: ; %bb.3: ; %bb49
-; GCN-NEXT: v_mov_b32_e32 v0, 1.0
-; GCN-NEXT: tbuffer_store_format_x v0, off, s[4:7], 1 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offset:4 glc
+; GCN-NEXT: v_mov_b32_e32 v1, 1.0
+; GCN-NEXT: tbuffer_store_format_x v1, off, s[4:7], 1 format:[BUF_DATA_FORMAT_32,BUF_NUM_FORMAT_FLOAT] offset:4 glc
; GCN-NEXT: .LBB1_4: ; %UnifiedReturnBlock
; GCN-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.chain.arg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.chain.arg.ll
index 2f06098222b70..6743fd1474778 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.chain.arg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.chain.arg.ll
@@ -841,6 +841,141 @@ define amdgpu_cs_chain void @set_inactive_chain_arg_last_vgpr(ptr addrspace(1) %
ret void
}
+; SimplifyDemandedBits must recurse into the active-value operand (operand 1)
+; of set_inactive_chain_arg. Only the low 16 bits are demanded, so the OR that
+; sets the upper 16 bits of the active value must be optimized away.
+define amdgpu_cs_chain void @set_inactive_chain_arg_active_demanded(ptr addrspace(1) %out, i32 %inactive, i32 %x) {
+; GISEL11-LABEL: set_inactive_chain_arg_active_demanded:
+; GISEL11: ; %bb.0:
+; GISEL11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL11-NEXT: s_or_saveexec_b32 s0, -1
+; GISEL11-NEXT: v_mov_b32_e32 v0, v10
+; GISEL11-NEXT: s_mov_b32 exec_lo, s0
+; GISEL11-NEXT: v_or_b32_e32 v1, 0xffff0000, v11
+; GISEL11-NEXT: s_or_saveexec_b32 s0, -1
+; GISEL11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GISEL11-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GISEL11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GISEL11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL11-NEXT: s_mov_b32 exec_lo, s0
+; GISEL11-NEXT: v_mov_b32_e32 v1, v0
+; GISEL11-NEXT: global_store_b32 v[8:9], v1, off
+; GISEL11-NEXT: s_endpgm
+;
+; DAGISEL11-LABEL: set_inactive_chain_arg_active_demanded:
+; DAGISEL11: ; %bb.0:
+; DAGISEL11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; DAGISEL11-NEXT: s_or_saveexec_b32 s0, -1
+; DAGISEL11-NEXT: v_mov_b32_e32 v0, v10
+; DAGISEL11-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL11-NEXT: s_or_saveexec_b32 s0, -1
+; DAGISEL11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; DAGISEL11-NEXT: v_cndmask_b32_e64 v0, v0, v11, s0
+; DAGISEL11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; DAGISEL11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; DAGISEL11-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL11-NEXT: v_mov_b32_e32 v1, v0
+; DAGISEL11-NEXT: global_store_b32 v[8:9], v1, off
+; DAGISEL11-NEXT: s_endpgm
+;
+; GISEL10-LABEL: set_inactive_chain_arg_active_demanded:
+; GISEL10: ; %bb.0:
+; GISEL10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL10-NEXT: s_or_saveexec_b32 s0, -1
+; GISEL10-NEXT: v_mov_b32_e32 v0, v10
+; GISEL10-NEXT: s_mov_b32 exec_lo, s0
+; GISEL10-NEXT: v_or_b32_e32 v1, 0xffff0000, v11
+; GISEL10-NEXT: s_or_saveexec_b32 s0, -1
+; GISEL10-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GISEL10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL10-NEXT: s_mov_b32 exec_lo, s0
+; GISEL10-NEXT: v_mov_b32_e32 v1, v0
+; GISEL10-NEXT: global_store_dword v[8:9], v1, off
+; GISEL10-NEXT: s_endpgm
+;
+; DAGISEL10-LABEL: set_inactive_chain_arg_active_demanded:
+; DAGISEL10: ; %bb.0:
+; DAGISEL10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; DAGISEL10-NEXT: s_or_saveexec_b32 s0, -1
+; DAGISEL10-NEXT: v_mov_b32_e32 v0, v10
+; DAGISEL10-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL10-NEXT: s_or_saveexec_b32 s0, -1
+; DAGISEL10-NEXT: v_cndmask_b32_e64 v0, v0, v11, s0
+; DAGISEL10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; DAGISEL10-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL10-NEXT: v_mov_b32_e32 v1, v0
+; DAGISEL10-NEXT: global_store_dword v[8:9], v1, off
+; DAGISEL10-NEXT: s_endpgm
+;
+; GISEL11_W64-LABEL: set_inactive_chain_arg_active_demanded:
+; GISEL11_W64: ; %bb.0:
+; GISEL11_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL11_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GISEL11_W64-NEXT: v_mov_b32_e32 v0, v10
+; GISEL11_W64-NEXT: s_mov_b64 exec, s[0:1]
+; GISEL11_W64-NEXT: v_or_b32_e32 v1, 0xffff0000, v11
+; GISEL11_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GISEL11_W64-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GISEL11_W64-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GISEL11_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GISEL11_W64-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL11_W64-NEXT: s_mov_b64 exec, s[0:1]
+; GISEL11_W64-NEXT: v_mov_b32_e32 v1, v0
+; GISEL11_W64-NEXT: global_store_b32 v[8:9], v1, off
+; GISEL11_W64-NEXT: s_endpgm
+;
+; DAGISEL11_W64-LABEL: set_inactive_chain_arg_active_demanded:
+; DAGISEL11_W64: ; %bb.0:
+; DAGISEL11_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; DAGISEL11_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; DAGISEL11_W64-NEXT: v_mov_b32_e32 v0, v10
+; DAGISEL11_W64-NEXT: s_mov_b64 exec, s[0:1]
+; DAGISEL11_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; DAGISEL11_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; DAGISEL11_W64-NEXT: v_cndmask_b32_e64 v0, v0, v11, s[0:1]
+; DAGISEL11_W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; DAGISEL11_W64-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; DAGISEL11_W64-NEXT: s_mov_b64 exec, s[0:1]
+; DAGISEL11_W64-NEXT: v_mov_b32_e32 v1, v0
+; DAGISEL11_W64-NEXT: global_store_b32 v[8:9], v1, off
+; DAGISEL11_W64-NEXT: s_endpgm
+;
+; GISEL10_W64-LABEL: set_inactive_chain_arg_active_demanded:
+; GISEL10_W64: ; %bb.0:
+; GISEL10_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL10_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GISEL10_W64-NEXT: v_mov_b32_e32 v0, v10
+; GISEL10_W64-NEXT: s_mov_b64 exec, s[0:1]
+; GISEL10_W64-NEXT: v_or_b32_e32 v1, 0xffff0000, v11
+; GISEL10_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GISEL10_W64-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GISEL10_W64-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL10_W64-NEXT: s_mov_b64 exec, s[0:1]
+; GISEL10_W64-NEXT: v_mov_b32_e32 v1, v0
+; GISEL10_W64-NEXT: global_store_dword v[8:9], v1, off
+; GISEL10_W64-NEXT: s_endpgm
+;
+; DAGISEL10_W64-LABEL: set_inactive_chain_arg_active_demanded:
+; DAGISEL10_W64: ; %bb.0:
+; DAGISEL10_W64-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; DAGISEL10_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; DAGISEL10_W64-NEXT: v_mov_b32_e32 v0, v10
+; DAGISEL10_W64-NEXT: s_mov_b64 exec, s[0:1]
+; DAGISEL10_W64-NEXT: s_or_saveexec_b64 s[0:1], -1
+; DAGISEL10_W64-NEXT: v_cndmask_b32_e64 v0, v0, v11, s[0:1]
+; DAGISEL10_W64-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; DAGISEL10_W64-NEXT: s_mov_b64 exec, s[0:1]
+; DAGISEL10_W64-NEXT: v_mov_b32_e32 v1, v0
+; DAGISEL10_W64-NEXT: global_store_dword v[8:9], v1, off
+; DAGISEL10_W64-NEXT: s_endpgm
+ %active = or i32 %x, -65536
+ %tmp = call i32 @llvm.amdgcn.set.inactive.chain.arg.i32(i32 %active, i32 %inactive) #0
+ %and = and i32 %tmp, 65535
+ %wwm = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %and)
+ store i32 %wwm, ptr addrspace(1) %out
+ ret void
+}
+
declare i32 @llvm.amdgcn.set.inactive.chain.arg.i32(i32, i32) #0
declare i64 @llvm.amdgcn.set.inactive.chain.arg.i64(i64, i64) #0
declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32 immarg, i32 immarg, i32 immarg, i1 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
index a98fca9d41039..11ffb09e1a921 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
@@ -505,6 +505,33 @@ define amdgpu_kernel void @set_inactive_p6(ptr addrspace(1) %out, ptr addrspace(
ret void
}
+; The result of set_inactive with a constant value operand must not be
+; constant-folded away: SimplifyDemandedBits must intersect the known bits of
+; both operands, so the inactive value (0x55555555) is preserved through the
+; demanded-bits mask.
+define amdgpu_kernel void @set_inactive_const_value_demanded(ptr addrspace(1) %out) {
+; GCN-LABEL: set_inactive_const_value_demanded:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_mov_b32_e32 v1, 0xaaaaaaaa
+; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: v_mov_b32_e32 v0, 0x55555555
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
+; GCN-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: s_endpgm
+ %tmp.0 = call i32 @llvm.amdgcn.set.inactive.i32(i32 -1431655766, i32 1431655765) #0
+ %and = and i32 %tmp.0, 65535
+ %tmp = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %and)
+ store i32 %tmp, ptr addrspace(1) %out
+ ret void
+}
+
declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #0
declare i64 @llvm.amdgcn.set.inactive.i64(i64, i64) #0
declare i32 @llvm.amdgcn.strict.wwm.i32(i32) #1
More information about the llvm-commits
mailing list