[llvm] [AMDGPU] Use first operand of zext to test first bit zero (PR #217195)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 19:51:00 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Shoreshen
<details>
<summary>Changes</summary>
Fix AMDGPU SAddr selection for zero-extended 32-bit offsets on targets with signed GVS VOffset.
For DAG, function `matchExtFromI32orI32` will return valid value if address can be `Addr:SGPR` + `Signed Offset:VGPR`, and the function is as follow:
```
static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
const SelectionDAG *DAG) {
if (Op.getValueType() == MVT::i32)
return Op;
if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
Op.getOpcode() != ISD::ANY_EXTEND &&
!(DAG->SignBitIsZero(Op) &&
Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND)))
return SDValue();
SDValue ExtSrc = Op.getOperand(0);
return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
}
```
Assuming:
1. Address is formed by addition, denote `Addr = add BASE, OFFSET`
2. Then `OFFSET` = `Op` under `AMDGPUDAGToDAGISel::SelectGlobalSAddr` calling `matchExtFromI32orI32`
3. `i64 Op = zext i32 Src` which means `i64 OFFSET = zext i32 Src`
4. `Src` some how have the value of `Src = 0x80000000`
5. `IsSigned` is true if we are running under signed offset arch
With above assumption, we have:
1. Fail in `Op.getValueType() == MVT::i32`
2. `Op.getOpcode() != ISD::SIGN_EXTEND`
3. `Op.getOpcode() != ISD::ANY_EXTEND`
4. `Op.getOpcode() == ISD::ZERO_EXTEND` and since `Op` is the result of zext, the highest bit of `Op` is zero so `DAG->SignBitIsZero(Op)` is also true
5. `!(DAG->SignBitIsZero(Op) && Op.getOpcode() == ISD::ZERO_EXTEND)` is false
Thus, it will pass the if check and return valid `SDValue`, which in the end create instructions such as `(LOAD/STORE_SADDR BASE, (ZEXT Src))` and `Src = 0x80000000`, which is actually addressing `BASE - 0x80000000`
---
Patch is 836.87 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217195.diff
8 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp (+2-2)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp (+3-2)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll (+2163-1133)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll (+3665-1178)
- (modified) llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll (+833-252)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll (+412-106)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll (+134-76)
- (modified) llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll (+52-21)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 744bde043e16c..53bd66ac2498d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2017,8 +2017,8 @@ static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
Op.getOpcode() != ISD::ANY_EXTEND &&
- !(DAG->SignBitIsZero(Op) &&
- Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND)))
+ !(Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND) &&
+ DAG->SignBitIsZero(Op.getOperand(0))))
return SDValue();
SDValue ExtSrc = Op.getOperand(0);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 3f0f3e115d906..c19ec2021d5f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -3735,8 +3735,9 @@ Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
m_SpecificICst(31))))
return Def->getOperand(1).getReg();
- if (VT->signBitIsZero(Reg))
- return matchZeroExtendFromS32(Reg);
+ Register ZextSrc = matchZeroExtendFromS32(Reg);
+ if (ZextSrc && VT->signBitIsZero(ZextSrc))
+ return ZextSrc;
return Register();
}
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index b0ebc333139a9..4651da4ffc12c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -8,19 +8,40 @@
; sure these are not incorrectly selected before gfx1250.
define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -54,19 +75,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; Maximum positive offset on gfx10
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:2047 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:2047 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:2047 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
; GFX950-SDAG: ; %bb.0:
@@ -101,19 +143,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i
; Maximum negative offset on gfx10
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:-2048 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:-2048 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
; GFX950-SDAG: ; %bb.0:
@@ -153,19 +216,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -199,19 +283,40 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn_2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn_2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_2048:
; GFX950-SDAG: ; %bb.0:
@@ -246,19 +351,40 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voff
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
; GFX950-SDAG: ; %bb.0:
@@ -312,14 +438,13 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -389,14 +514,13 @@ define am...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217195
More information about the llvm-commits
mailing list