[llvm] [AMDGPU] Use first operand of zext to test first bit zero (PR #217195)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 18 19:50:27 PDT 2026
https://github.com/Shoreshen created https://github.com/llvm/llvm-project/pull/217195
Fix AMDGPU SAddr selection for zero-extended 32-bit offsets on targets with signed GVS VOffset.
For DAG, function `matchExtFromI32orI32` will return valid value if address can be `Addr:SGPR` + `Signed Offset:VGPR`, and the function is as follow:
```
static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
const SelectionDAG *DAG) {
if (Op.getValueType() == MVT::i32)
return Op;
if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
Op.getOpcode() != ISD::ANY_EXTEND &&
!(DAG->SignBitIsZero(Op) &&
Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND)))
return SDValue();
SDValue ExtSrc = Op.getOperand(0);
return (ExtSrc.getValueType() == MVT::i32) ? ExtSrc : SDValue();
}
```
Assuming:
1. Address is formed by addition, denote `Addr = add BASE, OFFSET`
2. Then `OFFSET` = `Op` under `AMDGPUDAGToDAGISel::SelectGlobalSAddr` calling `matchExtFromI32orI32`
3. `i64 Op = zext i32 Src` which means `i64 OFFSET = zext i32 Src`
4. `Src` some how have the value of `Src = 0x80000000`
5. `IsSigned` is true if we are running under signed offset arch
With above assumption, we have:
1. Fail in `Op.getValueType() == MVT::i32`
2. `Op.getOpcode() != ISD::SIGN_EXTEND`
3. `Op.getOpcode() != ISD::ANY_EXTEND`
4. `Op.getOpcode() == ISD::ZERO_EXTEND` and since `Op` is the result of zext, the highest bit of `Op` is zero so `DAG->SignBitIsZero(Op)` is also true
5. `!(DAG->SignBitIsZero(Op) && Op.getOpcode() == ISD::ZERO_EXTEND)` is false
Thus, it will pass the if check and return valid `SDValue`, which in the end create instructions such as `(LOAD/STORE_SADDR BASE, (ZEXT Src))` and `Src = 0x80000000`, which is actually addressing `BASE - 0x80000000`
>From 681621dc969437ac7ca44722d79f6072def996b0 Mon Sep 17 00:00:00 2001
From: shore <shorshen at amd.com>
Date: Wed, 19 Aug 2026 10:27:10 +0800
Subject: [PATCH] use operand of zext to test first bit zero
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 4 +-
.../AMDGPU/AMDGPUInstructionSelector.cpp | 5 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 3296 +++++++----
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 4843 +++++++++++++----
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 1085 +++-
.../AMDGPU/llvm.amdgcn.av.load.b128.ll | 518 +-
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 210 +-
.../CodeGen/AMDGPU/load-saddr-offset-imm.ll | 73 +-
8 files changed, 7264 insertions(+), 2770 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 744bde043e16c..53bd66ac2498d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -2017,8 +2017,8 @@ static SDValue matchExtFromI32orI32(SDValue Op, bool IsSigned,
if (Op.getOpcode() != (IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND) &&
Op.getOpcode() != ISD::ANY_EXTEND &&
- !(DAG->SignBitIsZero(Op) &&
- Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND)))
+ !(Op.getOpcode() == (IsSigned ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND) &&
+ DAG->SignBitIsZero(Op.getOperand(0))))
return SDValue();
SDValue ExtSrc = Op.getOperand(0);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 3f0f3e115d906..c19ec2021d5f7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -3735,8 +3735,9 @@ Register AMDGPUInstructionSelector::matchSignExtendFromS32(Register Reg) const {
m_SpecificICst(31))))
return Def->getOperand(1).getReg();
- if (VT->signBitIsZero(Reg))
- return matchZeroExtendFromS32(Reg);
+ Register ZextSrc = matchZeroExtendFromS32(Reg);
+ if (ZextSrc && VT->signBitIsZero(ZextSrc))
+ return ZextSrc;
return Register();
}
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index b0ebc333139a9..4651da4ffc12c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -8,19 +8,40 @@
; sure these are not incorrectly selected before gfx1250.
define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -54,19 +75,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
; Maximum positive offset on gfx10
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:2047 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:2047 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:2047 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
; GFX950-SDAG: ; %bb.0:
@@ -101,19 +143,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i
; Maximum negative offset on gfx10
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:-2048 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v[2:3], v1 offset:-2048 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
; GFX950-SDAG: ; %bb.0:
@@ -153,19 +216,40 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -199,19 +283,40 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn_2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn_2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_2048:
; GFX950-SDAG: ; %bb.0:
@@ -246,19 +351,40 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voff
}
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b32 v0, v[2:3], v1 offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
; GFX950-SDAG: ; %bb.0:
@@ -312,14 +438,13 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -389,14 +514,13 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[0:1] offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v[0:1], v2 offset:42 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -467,14 +591,13 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset,
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -543,14 +666,13 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v0, v1, s[0:1] offset:42 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:42 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -675,13 +797,13 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -697,17 +819,17 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: .LBB10_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB10_2
; GFX1250-GISEL-NEXT: .LBB10_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: s_clause 0x1
@@ -865,16 +987,16 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -890,17 +1012,17 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: .LBB11_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB11_2
; GFX1250-GISEL-NEXT: .LBB11_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: s_clause 0x1
@@ -1058,11 +1180,11 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB12_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -1073,19 +1195,19 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB12_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB12_2
; GFX1250-GISEL-NEXT: .LBB12_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -1214,14 +1336,14 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB13_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -1232,19 +1354,19 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: .LBB13_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_swap_b64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB13_2
; GFX1250-GISEL-NEXT: .LBB13_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_store_b64 v0, v[4:5], off
; GFX1250-GISEL-NEXT: s_endpgm
;
@@ -1333,19 +1455,40 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_add_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_add_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_add_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_add_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_add_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_add_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -1379,21 +1522,42 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_add_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
-;
-; GFX950-SDAG-LABEL: flat_add_saddr_i32_rtn_neg128:
+; GFX1250-SDAG-LABEL: flat_add_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_add_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_add_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_add_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX950-SDAG-LABEL: flat_add_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, 0
@@ -1432,19 +1596,40 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_add_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_add_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_add_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_add_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_add_u32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_add_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -1477,19 +1662,40 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_add_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_add_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_add_u32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_add_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_add_u32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_add_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -1583,13 +1789,13 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1605,17 +1811,17 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: .LBB18_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB18_2
; GFX1250-GISEL-NEXT: .LBB18_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -1777,16 +1983,16 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1802,17 +2008,17 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: .LBB19_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB19_2
; GFX1250-GISEL-NEXT: .LBB19_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -1977,11 +2183,11 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB20_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -1992,19 +2198,19 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB20_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB20_2
; GFX1250-GISEL-NEXT: .LBB20_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]
@@ -2147,14 +2353,14 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB21_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -2165,19 +2371,19 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB21_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_add_u64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB21_2
; GFX1250-GISEL-NEXT: .LBB21_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[4:5]
@@ -2277,19 +2483,40 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_sub_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sub_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_sub_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sub_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_sub_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -2323,19 +2550,40 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_sub_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sub_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_sub_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sub_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_sub_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -2376,19 +2624,40 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_sub_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_sub_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_sub_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_sub_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_sub_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -2421,19 +2690,40 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_sub_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_sub_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_sub_u32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_sub_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_sub_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -2527,13 +2817,13 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2549,17 +2839,17 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: .LBB26_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB26_2
; GFX1250-GISEL-NEXT: .LBB26_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -2723,16 +3013,16 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2748,17 +3038,17 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: .LBB27_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB27_2
; GFX1250-GISEL-NEXT: .LBB27_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -2925,11 +3215,11 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB28_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -2940,19 +3230,19 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB28_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB28_2
; GFX1250-GISEL-NEXT: .LBB28_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[4:5]
@@ -3097,14 +3387,14 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB29_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -3115,19 +3405,19 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB29_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_sub_u64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB29_2
; GFX1250-GISEL-NEXT: .LBB29_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_sub_nc_u64_e32 v[0:1], v[0:1], v[4:5]
@@ -3229,19 +3519,40 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_and_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_and_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_and_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -3275,19 +3586,40 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_and_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_and_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_and_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -3328,19 +3660,40 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_and_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_and_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_and_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -3373,19 +3726,40 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_and_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_and_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_and_b32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_and_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_and_b32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_and_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -3480,13 +3854,13 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3502,17 +3876,17 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: .LBB34_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB34_2
; GFX1250-GISEL-NEXT: .LBB34_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3676,16 +4050,16 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3701,17 +4075,17 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: .LBB35_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB35_2
; GFX1250-GISEL-NEXT: .LBB35_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3878,11 +4252,11 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB36_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -3893,19 +4267,19 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB36_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB36_2
; GFX1250-GISEL-NEXT: .LBB36_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
@@ -4050,14 +4424,14 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB37_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -4068,19 +4442,19 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB37_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_and_b64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB37_2
; GFX1250-GISEL-NEXT: .LBB37_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, v0, v4
@@ -4181,19 +4555,40 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_or_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -4227,19 +4622,40 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i3
}
define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_or_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -4280,19 +4696,40 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
}
define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_or_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -4325,19 +4762,40 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_or_b32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_or_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_or_b32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_or_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -4432,13 +4890,13 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -4454,17 +4912,17 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: .LBB42_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB42_2
; GFX1250-GISEL-NEXT: .LBB42_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -4628,16 +5086,16 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -4653,17 +5111,17 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: .LBB43_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB43_2
; GFX1250-GISEL-NEXT: .LBB43_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -4830,11 +5288,11 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB44_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -4845,19 +5303,19 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
; GFX1250-GISEL-NEXT: .LBB44_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB44_2
; GFX1250-GISEL-NEXT: .LBB44_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
@@ -5002,14 +5460,14 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB45_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -5020,19 +5478,19 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: .LBB45_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_or_b64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB45_2
; GFX1250-GISEL-NEXT: .LBB45_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
@@ -5133,19 +5591,40 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -5179,19 +5658,40 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -5232,19 +5732,40 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -5277,19 +5798,40 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_xor_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_xor_b32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_xor_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -5384,13 +5926,13 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -5406,17 +5948,17 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: .LBB50_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB50_2
; GFX1250-GISEL-NEXT: .LBB50_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -5580,16 +6122,16 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -5605,17 +6147,17 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: .LBB51_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB51_2
; GFX1250-GISEL-NEXT: .LBB51_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -5782,11 +6324,11 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB52_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -5797,19 +6339,19 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB52_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[0:1], v[4:5] scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB52_2
; GFX1250-GISEL-NEXT: .LBB52_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
@@ -5954,14 +6496,14 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB53_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -5972,19 +6514,19 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB53_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: flat_atomic_xor_b64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB53_2
; GFX1250-GISEL-NEXT: .LBB53_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, v0, v4
@@ -6085,15 +6627,32 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_max_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_max_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_max_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_i32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -6121,15 +6680,32 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_max_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_max_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_i32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_max_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_i32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -6164,15 +6740,32 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_max_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_max_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_i32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_max_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_i32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -6199,15 +6792,32 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_max_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_max_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_i32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_max_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_i32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_max_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -6291,13 +6901,13 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -6311,15 +6921,15 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB58_5
; GFX1250-GISEL-NEXT: .LBB58_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB58_2
; GFX1250-GISEL-NEXT: .LBB58_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -6477,16 +7087,16 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -6500,15 +7110,15 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB59_5
; GFX1250-GISEL-NEXT: .LBB59_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB59_2
; GFX1250-GISEL-NEXT: .LBB59_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -6670,11 +7280,11 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB60_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -6683,18 +7293,18 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB60_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB60_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3]
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB60_2
; GFX1250-GISEL-NEXT: .LBB60_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]
@@ -6832,14 +7442,14 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB61_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -6848,18 +7458,18 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB61_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB61_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v0, v[4:5], s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_max_i64 v[2:3], v[4:5] offset:-128
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB61_2
; GFX1250-GISEL-NEXT: .LBB61_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_max_i64 v[0:1], v[0:1], v[4:5]
@@ -6957,15 +7567,32 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_min_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_min_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_min_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_i32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -6993,15 +7620,32 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_min_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_min_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_i32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_min_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_i32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -7036,15 +7680,32 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_min_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_min_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_i32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_min_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_i32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -7071,15 +7732,32 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_min_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_min_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_i32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_min_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_i32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_min_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -7163,13 +7841,13 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -7183,15 +7861,15 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB66_5
; GFX1250-GISEL-NEXT: .LBB66_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB66_2
; GFX1250-GISEL-NEXT: .LBB66_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -7349,16 +8027,16 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -7372,15 +8050,15 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB67_5
; GFX1250-GISEL-NEXT: .LBB67_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB67_2
; GFX1250-GISEL-NEXT: .LBB67_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -7542,11 +8220,11 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB68_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -7555,18 +8233,18 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB68_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB68_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3]
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB68_2
; GFX1250-GISEL-NEXT: .LBB68_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]
@@ -7704,14 +8382,14 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB69_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -7720,18 +8398,18 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB69_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB69_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v0, v[4:5], s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_min_i64 v[2:3], v[4:5] offset:-128
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB69_2
; GFX1250-GISEL-NEXT: .LBB69_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_min_i64 v[0:1], v[0:1], v[4:5]
@@ -7829,15 +8507,32 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umax_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_umax_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_umax_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -7865,15 +8560,32 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umax_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_umax_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_umax_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -7908,15 +8620,32 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
}
define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umax_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_umax_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_u32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_umax_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_u32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -7943,15 +8672,32 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umax_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_umax_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_max_u32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_umax_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_max_u32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umax_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -8035,13 +8781,13 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -8055,15 +8801,15 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB74_5
; GFX1250-GISEL-NEXT: .LBB74_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB74_2
; GFX1250-GISEL-NEXT: .LBB74_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -8221,16 +8967,16 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -8244,15 +8990,15 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB75_5
; GFX1250-GISEL-NEXT: .LBB75_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB75_2
; GFX1250-GISEL-NEXT: .LBB75_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -8414,11 +9160,11 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB76_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -8427,18 +9173,18 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB76_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB76_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3]
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB76_2
; GFX1250-GISEL-NEXT: .LBB76_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]
@@ -8576,14 +9322,14 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB77_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -8592,18 +9338,18 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: .LBB77_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB77_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v0, v[4:5], s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_max_u64 v[2:3], v[4:5] offset:-128
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB77_2
; GFX1250-GISEL-NEXT: .LBB77_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_max_u64 v[0:1], v[0:1], v[4:5]
@@ -8701,15 +9447,32 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umin_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_umin_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_umin_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -8737,15 +9500,32 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umin_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_umin_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_umin_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -8780,15 +9560,32 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
}
define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umin_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_umin_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_u32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_umin_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_u32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -8815,15 +9612,32 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_umin_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_umin_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_min_u32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_umin_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_min_u32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_umin_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -8907,13 +9721,13 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -8927,15 +9741,15 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB82_5
; GFX1250-GISEL-NEXT: .LBB82_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB82_2
; GFX1250-GISEL-NEXT: .LBB82_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -9093,16 +9907,16 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -9116,15 +9930,15 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-GISEL-NEXT: s_branch .LBB83_5
; GFX1250-GISEL-NEXT: .LBB83_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB83_2
; GFX1250-GISEL-NEXT: .LBB83_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -9286,11 +10100,11 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB84_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -9299,18 +10113,18 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB84_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB84_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3]
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB84_2
; GFX1250-GISEL-NEXT: .LBB84_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]
@@ -9448,14 +10262,14 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB85_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -9464,18 +10278,18 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: .LBB85_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB85_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v0, v[4:5], s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_min_u64 v[2:3], v[4:5] offset:-128
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB85_2
; GFX1250-GISEL-NEXT: .LBB85_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_min_u64 v[0:1], v[0:1], v[4:5]
@@ -9573,20 +10387,41 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
-; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -9622,20 +10457,41 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
-; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v0, v[2:3], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v0, v[0:1], v[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v0, v[0:1], v[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -9678,20 +10534,41 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32
}
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
-; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -9725,20 +10602,41 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
-; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:-128 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v3, v1
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v5, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -9835,14 +10733,14 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v0, v5
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v3 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -9858,7 +10756,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
; GFX1250-GISEL-NEXT: .LBB90_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[2:3], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -10042,17 +10940,17 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v5
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v4
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v5, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v3 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -10068,7 +10966,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
; GFX1250-GISEL-NEXT: .LBB91_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v5, v[6:9], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -10256,11 +11154,11 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB92_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -10271,19 +11169,19 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
; GFX1250-GISEL-NEXT: .LBB92_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:9] scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB92_2
; GFX1250-GISEL-NEXT: .LBB92_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -10438,14 +11336,14 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v4, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v4
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB93_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -10456,19 +11354,19 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
; GFX1250-GISEL-NEXT: .LBB93_3: ; %atomicrmw.global
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v0, v[6:9], s[2:3] offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[6:9] offset:-128 scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB93_2
; GFX1250-GISEL-NEXT: .LBB93_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
@@ -10577,15 +11475,32 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_inc_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_inc_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_inc_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_inc_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_inc_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -10613,15 +11528,32 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_inc_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_inc_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_inc_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_inc_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_inc_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -10656,14 +11588,30 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_inc_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_inc_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_inc_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_inc_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_inc_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -10688,14 +11636,30 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_inc_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_inc_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_inc_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_inc_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -10781,13 +11745,13 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -10802,16 +11766,16 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB98_5
; GFX1250-GISEL-NEXT: .LBB98_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB98_2
; GFX1250-GISEL-NEXT: .LBB98_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -10981,16 +11945,16 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11005,16 +11969,16 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB99_5
; GFX1250-GISEL-NEXT: .LBB99_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB99_2
; GFX1250-GISEL-NEXT: .LBB99_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -11185,11 +12149,11 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB100_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -11198,17 +12162,17 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB100_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB100_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[0:1], v[4:5] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB100_2
; GFX1250-GISEL-NEXT: .LBB100_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]
@@ -11353,14 +12317,14 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB101_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -11369,17 +12333,17 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB101_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB101_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_inc_u64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB101_2
; GFX1250-GISEL-NEXT: .LBB101_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], 1, v[0:1]
@@ -11484,15 +12448,32 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_dec_saddr_i32_rtn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_dec_saddr_i32_rtn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_dec_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_dec_saddr_i32_rtn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u32 v0, v[2:3], v1 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_dec_saddr_i32_rtn:
; GFX950-SDAG: ; %bb.0:
@@ -11520,15 +12501,32 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
}
define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_dec_saddr_i32_rtn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_dec_saddr_i32_rtn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_dec_u32 v0, v[0:1], v2 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_dec_saddr_i32_rtn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u32 v0, v[2:3], v1 offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX950-SDAG-LABEL: flat_dec_saddr_i32_rtn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -11563,14 +12561,30 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_dec_saddr_i32_nortn:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] scope:SCOPE_DEV
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_dec_saddr_i32_nortn:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_dec_u32 v[0:1], v2 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_dec_saddr_i32_nortn:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u32 v[2:3], v1 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_dec_saddr_i32_nortn:
; GFX950-SDAG: ; %bb.0:
@@ -11595,14 +12609,30 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
}
define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_dec_saddr_i32_nortn_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_dec_saddr_i32_nortn_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:-128 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_dec_saddr_i32_nortn_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u32 v[2:3], v1 offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: flat_dec_saddr_i32_nortn_neg128:
; GFX950-SDAG: ; %bb.0:
@@ -11690,13 +12720,13 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11711,16 +12741,16 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB106_5
; GFX1250-GISEL-NEXT: .LBB106_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v[2:3], v[4:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB106_2
; GFX1250-GISEL-NEXT: .LBB106_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -11896,16 +12926,16 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
-; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, 0xffffff80, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v6
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v7, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_bitop2_b32 v0, src_flat_scratch_base_hi, v7 bitop3:0x14
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v0, src_flat_scratch_base_hi, v3
; GFX1250-GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 0x4000000, v0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1250-GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11920,16 +12950,16 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: s_branch .LBB107_5
; GFX1250-GISEL-NEXT: .LBB107_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v3, v[4:5], s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr6
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v[6:7], v[4:5] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s1, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB107_2
; GFX1250-GISEL-NEXT: .LBB107_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v6
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -12106,11 +13136,11 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v2, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v2
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB108_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -12119,17 +13149,17 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
; GFX1250-GISEL-NEXT: .LBB108_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB108_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[0:1], v[4:5] scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB108_2
; GFX1250-GISEL-NEXT: .LBB108_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
@@ -12280,14 +13310,14 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v1, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, 0xffffff80, v1
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, -1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xffffff80, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v3, vcc_lo
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_xor_b32_e32 v1, src_flat_scratch_base_hi, v3
-; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v1
+; GFX1250-GISEL-NEXT: v_xor_b32_e32 v6, src_flat_scratch_base_hi, v1
+; GFX1250-GISEL-NEXT: v_cmpx_le_u32_e32 0x4000000, v6
; GFX1250-GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1250-GISEL-NEXT: s_cbranch_execnz .LBB109_3
; GFX1250-GISEL-NEXT: ; %bb.1: ; %Flow
@@ -12296,17 +13326,17 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: .LBB109_2: ; %atomicrmw.phi
; GFX1250-GISEL-NEXT: s_endpgm
; GFX1250-GISEL-NEXT: .LBB109_3: ; %atomicrmw.global
-; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_DEV
-; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr2
+; GFX1250-GISEL-NEXT: flat_atomic_dec_u64 v[2:3], v[4:5] offset:-128 scope:SCOPE_DEV
+; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1250-GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX1250-GISEL-NEXT: s_cbranch_execz .LBB109_2
; GFX1250-GISEL-NEXT: .LBB109_4: ; %atomicrmw.private
-; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[2:3]
-; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v0, src_flat_scratch_base_lo, v2
+; GFX1250-GISEL-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-GISEL-NEXT: v_subrev_nc_u32_e32 v2, src_flat_scratch_base_lo, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc_lo
; GFX1250-GISEL-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-GISEL-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[0:1], v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 990b13ca76fb8..8efb62f0a15e5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -514,15 +514,45 @@ define amdgpu_ps float @flat_load_saddr_i8_offset_neg0x100000001(ptr inreg %sbas
; Basic pattern, no immediate offset.
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i8, ptr %gep0
@@ -533,15 +563,45 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr(ptr inreg %sbase, i32 %voff
; Maximum positive offset on gfx1250
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388607
@@ -612,324 +672,197 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388608(ptr inreg %s
; Maximum negative offset on gfx1250
define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388608(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388608
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388608
- %load = load i8, ptr %gep1
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; Maximum negative offset on gfx1250 - 1
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388607(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-8388607
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388607
- %load = load i8, ptr %gep1
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:8388607
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 8388607
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 %zext.offset
- %load = load i8, ptr %gep1
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; pointer addressing done in integers
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %sbase.as.int = ptrtoint ptr %sbase to i64
- %add = add i64 %sbase.as.int, %zext.offset
- %dirty.gep = inttoptr i64 %add to ptr
- %load = load i8, ptr %dirty.gep
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; zext forced to LHS of addressing expression
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %sbase.as.int = ptrtoint ptr %sbase to i64
- %add = add i64 %zext.offset, %sbase.as.int
- %dirty.gep = inttoptr i64 %add to ptr
- %load = load i8, ptr %dirty.gep
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; zext forced to LHS of addressing expression, with immediate offset
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %sbase.as.int = ptrtoint ptr %sbase to i64
- %add = add i64 %zext.offset, %sbase.as.int
- %add.immoffset = add i64 %add, 128
- %dirty.gep = inttoptr i64 %add.immoffset to ptr
- %load = load i8, ptr %dirty.gep
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; zext forced to LHS of addressing expression, with immediate offset in non-canonical position
-define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %sbase.as.int = ptrtoint ptr %sbase to i64
- %add.immoffset = add i64 %sbase.as.int, 128
- %add = add i64 %zext.offset, %add.immoffset
- %dirty.gep = inttoptr i64 %add to ptr
- %load = load i8, ptr %dirty.gep
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; --------------------------------------------------------------------------------
-; Uniformity edge cases
-; --------------------------------------------------------------------------------
-
- at ptr.in.lds = internal addrspace(3) global ptr undef
-
-; Base pointer is uniform, but also in VGPRs
-define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-8388608
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:-8388608
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388608:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-NOECC-NEXT: s_wait_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-NOECC-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:-8388608
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
- %sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %load = load i8, ptr %gep0
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388608
+ %load = load i8, ptr %gep1
%zext = zext i8 %load to i32
%to.vgpr = bitcast i32 %zext to float
ret float %to.vgpr
}
-; Base pointer is uniform, but also in VGPRs, with imm offset
-define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
+; Maximum negative offset on gfx1250 - 1
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_neg8388607(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1] offset:42
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-8388607
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:42
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:-8388607
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_offset_neg8388607:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
-; GFX1250-NOECC-NEXT: s_wait_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-NOECC-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1] offset:42
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:-8388607
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
- %sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -8388607
%load = load i8, ptr %gep1
%zext = zext i8 %load to i32
%to.vgpr = bitcast i32 %zext to float
ret float %to.vgpr
}
-; Both 64-bit base and 32-bit offset are scalar
-define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset(ptr inreg %sbase, i32 inreg %soffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s4
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %load = load i8, ptr %gep0
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; Both 64-bit base and 32-bit offset are scalar, with immediate offset.
-define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset_immoffset(ptr inreg %sbase, i32 inreg %soffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s4
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-24
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -24
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_offset_8388607_gep_order:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 8388607
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 %zext.offset
%load = load i8, ptr %gep1
%zext = zext i8 %load to i32
%to.vgpr = bitcast i32 %zext to float
ret float %to.vgpr
}
-; Both components uniform, zext forced to LHS of addressing expression
-define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 inreg %soffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s4
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
+; pointer addressing done in integers
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
%sbase.as.int = ptrtoint ptr %sbase to i64
- %add = add i64 %zext.offset, %sbase.as.int
+ %add = add i64 %sbase.as.int, %zext.offset
%dirty.gep = inttoptr i64 %add to ptr
%load = load i8, ptr %dirty.gep
%zext = zext i8 %load to i32
@@ -937,365 +870,977 @@ define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr in
ret float %to.vgpr
}
-; Both components uniform, zext forced to LHS of addressing expression, with immediate offset
-define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 inreg %soffset) {
-; GFX1250-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_mov_b32_e32 v0, s4
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
- %sbase.as.int = ptrtoint ptr %sbase to i64
- %add = add i64 %zext.offset, %sbase.as.int
- %add.immoffset = add i64 %add, 128
- %dirty.gep = inttoptr i64 %add.immoffset to ptr
- %load = load i8, ptr %dirty.gep
- %zext = zext i8 %load to i32
- %to.vgpr = bitcast i32 %zext to float
- ret float %to.vgpr
-}
-
-; divergent 64-bit base, 32-bit scalar offset.
-define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffset) {
-; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32:
+; zext forced to LHS of addressing expression
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32:
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
-; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
- %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset
- %load = load i8, ptr %gep0
+ %zext.offset = zext i32 %voffset to i64
+ %sbase.as.int = ptrtoint ptr %sbase to i64
+ %add = add i64 %zext.offset, %sbase.as.int
+ %dirty.gep = inttoptr i64 %add to ptr
+ %load = load i8, ptr %dirty.gep
%zext = zext i8 %load to i32
%to.vgpr = bitcast i32 %zext to float
ret float %to.vgpr
}
-; divergent 64-bit base, 32-bit scalar offset, with imm offset
-define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i32 inreg %soffset) {
-; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; zext forced to LHS of addressing expression, with immediate offset
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 0x80, v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x80, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset0:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
-; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], 0x80, v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %soffset to i64
- %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388607
- %load = load i8, ptr %gep1
+ %zext.offset = zext i32 %voffset to i64
+ %sbase.as.int = ptrtoint ptr %sbase to i64
+ %add = add i64 %zext.offset, %sbase.as.int
+ %add.immoffset = add i64 %add, 128
+ %dirty.gep = inttoptr i64 %add.immoffset to ptr
+ %load = load i8, ptr %dirty.gep
%zext = zext i8 %load to i32
%to.vgpr = bitcast i32 %zext to float
ret float %to.vgpr
}
-; --------------------------------------------------------------------------------
-; Natural addressing shifts with restricted range
-; --------------------------------------------------------------------------------
-
-; Cannot push the shift into 32-bits, and cannot match.
-define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase, ptr %voffset.ptr) {
-; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scale_offset
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %voffset = load i32, ptr %voffset.ptr
- %zext.offset = zext i32 %voffset to i64
- %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
- %load = load float, ptr %gep
- ret float %load
-}
-
-; Cannot push the shift into 32-bits, with an immediate offset.
-define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inreg %sbase, ptr %voffset.ptr) {
-; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing_immoffset:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %voffset = load i32, ptr %voffset.ptr
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 128
- %load = load float, ptr %gep1
- ret float %load
-}
-
-; Range is sufficiently restricted to push the shift into 32-bits.
-define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, ptr %voffset.ptr) {
-; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scale_offset
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
- %zext.offset = zext i32 %voffset to i64
- %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
- %load = load float, ptr %gep
- ret float %load
-}
-
-; Range is sufficiently restricted to push the shift into 32-bits, with an imm offset
-define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg %sbase, ptr %voffset.ptr) {
-; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:400 scale_offset
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds float, ptr %gep0, i64 100
- %load = load float, ptr %gep1
- ret float %load
-}
-
-; Range is 1 beyond the limit where we can move the shift into 32-bits.
-define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_too_large(ptr inreg %sbase, ptr %voffset.ptr) {
-; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_too_large:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scale_offset
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %voffset = load i32, ptr %voffset.ptr, !range !1, !noundef !{}
+; zext forced to LHS of addressing expression, with immediate offset in non-canonical position
+define amdgpu_ps float @flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], 0x80, v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0x80, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offset1:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], 0x80, v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
- %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
- %load = load float, ptr %gep
- ret float %load
+ %sbase.as.int = ptrtoint ptr %sbase to i64
+ %add.immoffset = add i64 %sbase.as.int, 128
+ %add = add i64 %zext.offset, %add.immoffset
+ %dirty.gep = inttoptr i64 %add to ptr
+ %load = load i8, ptr %dirty.gep
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
}
; --------------------------------------------------------------------------------
-; Stress various type loads
+; Uniformity edge cases
; --------------------------------------------------------------------------------
-define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_i16:
+ at ptr.in.lds = internal addrspace(3) global ptr undef
+
+; Base pointer is uniform, but also in VGPRs
+define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16:
+; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16:
-; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16:
-; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-NOECC-NEXT: s_wait_dscnt 0x0
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %load = load i16, ptr %gep0
- %cast.load = bitcast i16 %load to half
- ret half %cast.load
+ %load = load i8, ptr %gep0
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
}
-define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_i16_immneg128:
+; Base pointer is uniform, but also in VGPRs, with imm offset
+define amdgpu_ps float @flat_load_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:42
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-GISEL-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:42
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_immneg128:
-; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_immneg128:
-; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-NOECC-LABEL: flat_load_saddr_uniform_ptr_in_vgprs_immoffset:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: ds_load_b64 v[2:3], v1
+; GFX1250-NOECC-NEXT: s_wait_dscnt 0x0
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:42
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %load = load i16, ptr %gep1
- %cast.load = bitcast i16 %load to half
- ret half %cast.load
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 42
+ %load = load i8, ptr %gep1
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
}
-define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_f16:
+; Both 64-bit base and 32-bit offset are scalar
+define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset(ptr inreg %sbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_uniform_offset:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s5, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[4:5]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_f16:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_uniform_offset:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, s4
+; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16:
-; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_uniform_offset:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NOECC-NEXT: s_mov_b32 s5, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[4:5]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %load = load i8, ptr %gep0
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; Both 64-bit base and 32-bit offset are scalar, with immediate offset.
+define amdgpu_ps float @flat_load_saddr_i8_zext_uniform_offset_immoffset(ptr inreg %sbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s5, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[4:5]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1] offset:-24
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, s4
+; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[0:1] offset:-24
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_uniform_offset_immoffset:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NOECC-NEXT: s_mov_b32 s5, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[4:5]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1] offset:-24
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -24
+ %load = load i8, ptr %gep1
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; Both components uniform, zext forced to LHS of addressing expression
+define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr inreg %sbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s5, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[4:5], s[2:3]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, s4
+; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NOECC-NEXT: s_mov_b32 s5, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: s_add_nc_u64 s[0:1], s[4:5], s[2:3]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %sbase.as.int = ptrtoint ptr %sbase to i64
+ %add = add i64 %zext.offset, %sbase.as.int
+ %dirty.gep = inttoptr i64 %add to ptr
+ %load = load i8, ptr %dirty.gep
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; Both components uniform, zext forced to LHS of addressing expression, with immediate offset
+define amdgpu_ps float @flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0(ptr inreg %sbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s5, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[4:5], s[2:3]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v0, s[0:1] offset:128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-GISEL-NEXT: s_add_co_u32 s0, s2, s4
+; GFX1250-GISEL-NEXT: s_add_co_ci_u32 s1, s3, 0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[0:1] offset:128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offset0:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NOECC-NEXT: s_mov_b32 s5, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: s_add_nc_u64 s[0:1], s[4:5], s[2:3]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v0, s[0:1] offset:128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %sbase.as.int = ptrtoint ptr %sbase to i64
+ %add = add i64 %zext.offset, %sbase.as.int
+ %add.immoffset = add i64 %add, 128
+ %dirty.gep = inttoptr i64 %add.immoffset to ptr
+ %load = load i8, ptr %dirty.gep
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; divergent 64-bit base, 32-bit scalar offset.
+define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32(ptr %vbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset
+ %load = load i8, ptr %gep0
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; divergent 64-bit base, 32-bit scalar offset, with imm offset
+define amdgpu_ps float @flat_load_i8_vgpr64_sgpr32_offset_8388607(ptr %vbase, i32 inreg %soffset) {
+; GFX1250-SDAG-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_mov_b32 s3, 0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_i8_vgpr64_sgpr32_offset_8388607:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: s_mov_b32 s3, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:8388607
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %soffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %vbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 8388607
+ %load = load i8, ptr %gep1
+ %zext = zext i8 %load to i32
+ %to.vgpr = bitcast i32 %zext to float
+ ret float %to.vgpr
+}
+
+; --------------------------------------------------------------------------------
+; Natural addressing shifts with restricted range
+; --------------------------------------------------------------------------------
+
+; Cannot push the shift into 32-bits, and cannot match.
+define amdgpu_ps float @flat_load_saddr_f32_natural_addressing(ptr inreg %sbase, ptr %voffset.ptr) {
+; GFX1250-LABEL: flat_load_saddr_f32_natural_addressing:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[2:3]
+; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+ %voffset = load i32, ptr %voffset.ptr
+ %zext.offset = zext i32 %voffset to i64
+ %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
+ %load = load float, ptr %gep
+ ret float %load
+}
+
+; Cannot push the shift into 32-bits, with an immediate offset.
+define amdgpu_ps float @flat_load_saddr_f32_natural_addressing_immoffset(ptr inreg %sbase, ptr %voffset.ptr) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_f32_natural_addressing_immoffset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f32_natural_addressing_immoffset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_f32_natural_addressing_immoffset:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_xcnt 0x0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %voffset = load i32, ptr %voffset.ptr
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 128
+ %load = load float, ptr %gep1
+ ret float %load
+}
+
+; Range is sufficiently restricted to push the shift into 32-bits.
+define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range(ptr inreg %sbase, ptr %voffset.ptr) {
+; GFX1250-SDAG-LABEL: flat_load_f32_saddr_zext_vgpr_range:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_f32_saddr_zext_vgpr_range:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_f32_saddr_zext_vgpr_range:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
+ %zext.offset = zext i32 %voffset to i64
+ %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
+ %load = load float, ptr %gep
+ ret float %load
+}
+
+; Range is sufficiently restricted to push the shift into 32-bits, with an imm offset
+define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_imm_offset(ptr inreg %sbase, ptr %voffset.ptr) {
+; GFX1250-SDAG-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v2, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_f32_saddr_zext_vgpr_range_imm_offset:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:400
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %voffset = load i32, ptr %voffset.ptr, !range !0, !noundef !{}
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds float, ptr %gep0, i64 100
+ %load = load float, ptr %gep1
+ ret float %load
+}
+
+; Range is 1 beyond the limit where we can move the shift into 32-bits.
+define amdgpu_ps float @flat_load_f32_saddr_zext_vgpr_range_too_large(ptr inreg %sbase, ptr %voffset.ptr) {
+; GFX1250-LABEL: flat_load_f32_saddr_zext_vgpr_range_too_large:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scale_offset
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: ; return to shader part epilog
+ %voffset = load i32, ptr %voffset.ptr, !range !1, !noundef !{}
+ %zext.offset = zext i32 %voffset to i64
+ %gep = getelementptr inbounds float, ptr %sbase, i64 %zext.offset
+ %load = load float, ptr %gep
+ ret float %load
+}
+
+; --------------------------------------------------------------------------------
+; Stress various type loads
+; --------------------------------------------------------------------------------
+
+define amdgpu_ps half @flat_load_saddr_i16(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16:
+; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16:
+; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %load = load i16, ptr %gep0
+ %cast.load = bitcast i16 %load to half
+ ret half %cast.load
+}
+
+define amdgpu_ps half @flat_load_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:-128
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_immneg128:
+; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %load = load i16, ptr %gep1
+ %cast.load = bitcast i16 %load to half
+ ret half %cast.load
+}
+
+define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_f16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16:
+; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1]
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1305,89 +1850,165 @@ define amdgpu_ps half @flat_load_saddr_f16(ptr inreg %sbase, i32 %voffset) {
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %load = load half, ptr %gep0
+ ret half %load
+}
+
+define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_f16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16_immneg128:
+; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v[0:1] offset:-128
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_f16_immneg128:
+; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %load = load half, ptr %gep0
- ret half %load
+ %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
+ %load = load half, ptr %gep1
+ ret half %load
+}
+
+define amdgpu_ps float @flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
+ %zext.offset = zext i32 %voffset to i64
+ %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
+ %load = load i32, ptr %gep0
+ %cast.load = bitcast i32 %load to float
+ ret float %cast.load
}
-define amdgpu_ps half @flat_load_saddr_f16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-SDAG-LABEL: flat_load_saddr_f16_immneg128:
+define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_i32_immneg128:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_f16_immneg128:
+; GFX1250-GISEL-LABEL: flat_load_saddr_i32_immneg128:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
-; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_f16_immneg128:
-; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_f16_immneg128:
-; GFX1250-NOECC-SDAG-FAKE16: ; %bb.0:
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_nop
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
- %load = load half, ptr %gep1
- ret half %load
-}
-
-define amdgpu_ps float @flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
- %zext.offset = zext i32 %voffset to i64
- %gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
- %load = load i32, ptr %gep0
- %cast.load = bitcast i32 %load to float
- ret float %cast.load
-}
-
-define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-NOECC-LABEL: flat_load_saddr_i32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1397,15 +2018,45 @@ define amdgpu_ps float @flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps float @flat_load_saddr_f32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_f32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load float, ptr %gep0
@@ -1413,15 +2064,45 @@ define amdgpu_ps float @flat_load_saddr_f32(ptr inreg %sbase, i32 %voffset) {
}
define amdgpu_ps float @flat_load_saddr_f32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_f32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_f32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_f32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1430,15 +2111,45 @@ define amdgpu_ps float @flat_load_saddr_f32_immneg128(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps <2 x half> @flat_load_saddr_v2i16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x i16>, ptr %gep0
@@ -1447,15 +2158,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2i16(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <2 x half> @flat_load_saddr_v2i16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1465,15 +2206,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2i16_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <2 x half> @flat_load_saddr_v2f16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2f16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2f16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2f16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x half>, ptr %gep0
@@ -1481,15 +2252,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2f16(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <2 x half> @flat_load_saddr_v2f16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2f16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2f16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2f16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2f16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1498,15 +2299,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_v2f16_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <2 x half> @flat_load_saddr_p3(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_p3:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_p3:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_p3:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_p3:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load ptr addrspace(3), ptr %gep0
@@ -1516,15 +2347,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_p3(ptr inreg %sbase, i32 %voffset)
}
define amdgpu_ps <2 x half> @flat_load_saddr_p3_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_p3_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_p3_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_p3_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_p3_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1535,15 +2396,45 @@ define amdgpu_ps <2 x half> @flat_load_saddr_p3_immneg128(ptr inreg %sbase, i32
}
define amdgpu_ps <2 x float> @flat_load_saddr_f64(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_f64:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_f64:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load double, ptr %gep0
@@ -1552,15 +2443,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_f64(ptr inreg %sbase, i32 %voffset
}
define amdgpu_ps <2 x float> @flat_load_saddr_f64_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_f64_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_f64_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_f64_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_f64_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1570,15 +2491,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_f64_immneg128(ptr inreg %sbase, i3
}
define amdgpu_ps <2 x float> @flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i64:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i64:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i64, ptr %gep0
@@ -1587,15 +2538,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset
}
define amdgpu_ps <2 x float> @flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i64_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i64_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i64_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i64_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1605,15 +2586,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_i64_immneg128(ptr inreg %sbase, i3
}
define amdgpu_ps <2 x float> @flat_load_saddr_v2f32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2f32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x float>, ptr %gep0
@@ -1621,15 +2632,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2f32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <2 x float> @flat_load_saddr_v2f32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2f32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2f32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2f32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2f32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1638,15 +2679,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2f32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <2 x float> @flat_load_saddr_v2i32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x i32>, ptr %gep0
@@ -1655,15 +2726,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2i32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <2 x float> @flat_load_saddr_v2i32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1673,32 +2774,92 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v2i32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <2 x float> @flat_load_saddr_v4i16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4i16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <4 x i16>, ptr %gep0
%cast.load = bitcast <4 x i16> %load to <2 x float>
ret <2 x float> %cast.load
-}
-
-define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4i16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+}
+
+define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase, i32 %voffset) {
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4i16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4i16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4i16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1708,15 +2869,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4i16_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <2 x float> @flat_load_saddr_v4f16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4f16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4f16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4f16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <4 x half>, ptr %gep0
@@ -1725,15 +2916,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4f16(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <2 x float> @flat_load_saddr_v4f16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4f16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4f16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4f16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4f16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1743,15 +2964,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_v4f16_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <2 x float> @flat_load_saddr_p1(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_p1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_p1:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_p1:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_p1:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load ptr, ptr %gep0
@@ -1761,15 +3012,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_p1(ptr inreg %sbase, i32 %voffset)
}
define amdgpu_ps <2 x float> @flat_load_saddr_p1_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_p1_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_p1_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_p1_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_p1_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1780,15 +3061,45 @@ define amdgpu_ps <2 x float> @flat_load_saddr_p1_immneg128(ptr inreg %sbase, i32
}
define amdgpu_ps <3 x float> @flat_load_saddr_v3f32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v3f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v3f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v3f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v3f32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <3 x float>, ptr %gep0
@@ -1796,15 +3107,45 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3f32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <3 x float> @flat_load_saddr_v3f32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v3f32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v3f32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v3f32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v3f32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1813,15 +3154,45 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3f32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <3 x float> @flat_load_saddr_v3i32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v3i32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v3i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v3i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v3i32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <3 x i32>, ptr %gep0
@@ -1830,15 +3201,45 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3i32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <3 x float> @flat_load_saddr_v3i32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v3i32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v3i32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v3i32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v3i32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1848,15 +3249,45 @@ define amdgpu_ps <3 x float> @flat_load_saddr_v3i32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <6 x half> @flat_load_saddr_v6f16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v6f16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v6f16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v6f16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v6f16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <6 x half>, ptr %gep0
@@ -1864,15 +3295,45 @@ define amdgpu_ps <6 x half> @flat_load_saddr_v6f16(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <6 x half> @flat_load_saddr_v6f16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v6f16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b96 v[0:2], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v6f16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v6f16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v6f16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b96 v[0:2], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1881,15 +3342,45 @@ define amdgpu_ps <6 x half> @flat_load_saddr_v6f16_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4f32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4f32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4f32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <4 x float>, ptr %gep0
@@ -1897,15 +3388,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4f32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4f32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4f32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4f32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4f32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4f32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1914,15 +3435,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4f32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4i32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4i32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4i32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <4 x i32>, ptr %gep0
@@ -1931,15 +3482,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4i32(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4i32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4i32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4i32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4i32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4i32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1949,15 +3530,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4i32_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <4 x float> @flat_load_saddr_v2i64(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i64:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i64:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x i64>, ptr %gep0
@@ -1966,15 +3577,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2i64(ptr inreg %sbase, i32 %voffs
}
define amdgpu_ps <4 x float> @flat_load_saddr_v2i64_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2i64_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2i64_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2i64_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2i64_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1984,15 +3625,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2i64_immneg128(ptr inreg %sbase,
}
define amdgpu_ps <4 x float> @flat_load_saddr_i128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i128, ptr %gep0
@@ -2001,15 +3672,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_i128(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <4 x float> @flat_load_saddr_i128_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_i128_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_i128_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_i128_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_i128_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2019,15 +3720,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_i128_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <4 x float> @flat_load_saddr_v2p1(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2p1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2p1:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2p1:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2p1:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <2 x ptr>, ptr %gep0
@@ -2037,15 +3768,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2p1(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <4 x float> @flat_load_saddr_v2p1_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v2p1_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v2p1_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v2p1_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v2p1_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2056,15 +3817,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v2p1_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4p3(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4p3:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4p3:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4p3:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4p3:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load <4 x ptr addrspace(3)>, ptr %gep0
@@ -2074,15 +3865,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4p3(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps <4 x float> @flat_load_saddr_v4p3_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_load_saddr_v4p3_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b128 v[0:3], v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_load_saddr_v4p3_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_load_saddr_v4p3_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_load_saddr_v4p3_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b128 v[0:3], v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2097,15 +3918,45 @@ define amdgpu_ps <4 x float> @flat_load_saddr_v4p3_immneg128(ptr inreg %sbase, i
; --------------------------------------------------------------------------------
define amdgpu_ps float @flat_sextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_sextload_saddr_i8:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sextload_saddr_i8:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sextload_saddr_i8:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_sextload_saddr_i8:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i8, ptr %gep0
@@ -2115,15 +3966,45 @@ define amdgpu_ps float @flat_sextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
}
define amdgpu_ps float @flat_sextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_sextload_saddr_i8_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sextload_saddr_i8_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sextload_saddr_i8_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_sextload_saddr_i8_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2134,15 +4015,45 @@ define amdgpu_ps float @flat_sextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %
}
define amdgpu_ps float @flat_sextload_saddr_i16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_sextload_saddr_i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sextload_saddr_i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_i16 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sextload_saddr_i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_i16 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_sextload_saddr_i16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_i16 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i16, ptr %gep0
@@ -2152,15 +4063,45 @@ define amdgpu_ps float @flat_sextload_saddr_i16(ptr inreg %sbase, i32 %voffset)
}
define amdgpu_ps float @flat_sextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_sextload_saddr_i16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_i16 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_sextload_saddr_i16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_i16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_sextload_saddr_i16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_i16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_sextload_saddr_i16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_i16 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2171,15 +4112,45 @@ define amdgpu_ps float @flat_sextload_saddr_i16_immneg128(ptr inreg %sbase, i32
}
define amdgpu_ps float @flat_zextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_zextload_saddr_i8:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_zextload_saddr_i8:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_zextload_saddr_i8:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_zextload_saddr_i8:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i8, ptr %gep0
@@ -2189,15 +4160,45 @@ define amdgpu_ps float @flat_zextload_saddr_i8(ptr inreg %sbase, i32 %voffset) {
}
define amdgpu_ps float @flat_zextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_zextload_saddr_i8_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_zextload_saddr_i8_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_zextload_saddr_i8_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_zextload_saddr_i8_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2208,15 +4209,45 @@ define amdgpu_ps float @flat_zextload_saddr_i8_immneg128(ptr inreg %sbase, i32 %
}
define amdgpu_ps float @flat_zextload_saddr_i16(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_zextload_saddr_i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_zextload_saddr_i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_zextload_saddr_i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_zextload_saddr_i16:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load i16, ptr %gep0
@@ -2226,15 +4257,45 @@ define amdgpu_ps float @flat_zextload_saddr_i16(ptr inreg %sbase, i32 %voffset)
}
define amdgpu_ps float @flat_zextload_saddr_i16_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: flat_zextload_saddr_i16_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: flat_zextload_saddr_i16_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: flat_zextload_saddr_i16_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: flat_zextload_saddr_i16_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2249,17 +4310,51 @@ define amdgpu_ps float @flat_zextload_saddr_i16_immneg128(ptr inreg %sbase, i32
; --------------------------------------------------------------------------------
define amdgpu_ps float @atomic_flat_load_saddr_i32(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: atomic_flat_load_saddr_i32:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: atomic_flat_load_saddr_i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: atomic_flat_load_saddr_i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: atomic_flat_load_saddr_i32:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load atomic i32, ptr %gep0 seq_cst, align 4
@@ -2268,17 +4363,51 @@ define amdgpu_ps float @atomic_flat_load_saddr_i32(ptr inreg %sbase, i32 %voffse
}
define amdgpu_ps float @atomic_flat_load_saddr_i32_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: atomic_flat_load_saddr_i32_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v0, s[2:3] offset:-128 scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: atomic_flat_load_saddr_i32_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: atomic_flat_load_saddr_i32_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: atomic_flat_load_saddr_i32_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b32 v0, v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2288,17 +4417,51 @@ define amdgpu_ps float @atomic_flat_load_saddr_i32_immneg128(ptr inreg %sbase, i
}
define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: atomic_flat_load_saddr_i64:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: atomic_flat_load_saddr_i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: atomic_flat_load_saddr_i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: atomic_flat_load_saddr_i64:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%load = load atomic i64, ptr %gep0 seq_cst, align 8
@@ -2307,17 +4470,51 @@ define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64(ptr inreg %sbase, i32 %
}
define amdgpu_ps <2 x float> @atomic_flat_load_saddr_i64_immneg128(ptr inreg %sbase, i32 %voffset) {
-; GFX1250-LABEL: atomic_flat_load_saddr_i64_immneg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b64 v[0:1], v0, s[2:3] offset:-128 scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: atomic_flat_load_saddr_i64_immneg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: atomic_flat_load_saddr_i64_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1250-NOECC-LABEL: atomic_flat_load_saddr_i64_immneg128:
+; GFX1250-NOECC: ; %bb.0:
+; GFX1250-NOECC-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NOECC-NEXT: v_nop
+; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_b64 v[0:1], v[0:1] offset:-128 scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NOECC-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NOECC-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2337,7 +4534,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
@@ -2347,7 +4547,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2360,7 +4564,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2370,7 +4578,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi(ptr inreg %sbase
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
@@ -2388,7 +4599,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: ; return to shader part epilog
;
@@ -2398,7 +4612,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2411,7 +4629,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2421,7 +4643,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_undef_hi_immneg128(ptr in
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v0, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
@@ -2440,7 +4665,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -2451,7 +4679,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -2462,7 +4694,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -2473,7 +4708,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -2485,7 +4724,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi(ptr inreg %sbase,
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v[2:3]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-NEXT: ; return to shader part epilog
@@ -2504,7 +4745,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -2515,7 +4759,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
@@ -2526,7 +4774,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -2537,7 +4788,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -2549,7 +4804,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zero_hi_immneg128(ptr inr
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v[2:3] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-NEXT: ; return to shader part epilog
@@ -2569,9 +4826,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
@@ -2580,7 +4840,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -2591,10 +4855,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2604,9 +4871,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2623,9 +4893,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
@@ -2634,7 +4907,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -2645,10 +4922,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2658,9 +4938,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_b16 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_b16 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2678,9 +4961,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
@@ -2689,7 +4975,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -2700,10 +4990,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2713,9 +5006,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_u8 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_u8 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2733,9 +5029,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
@@ -2744,7 +5043,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -2755,10 +5058,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2768,9 +5074,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_u8 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_u8 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2789,9 +5098,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
@@ -2800,7 +5112,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2813,10 +5129,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2826,7 +5145,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2839,9 +5162,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_i8 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_i8 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2859,9 +5185,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
@@ -2870,7 +5199,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2883,10 +5216,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
-; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2896,7 +5232,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2909,9 +5249,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_i8 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_i8 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -2934,7 +5277,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -2945,7 +5291,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2958,7 +5309,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -2969,7 +5323,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -2980,7 +5338,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi(ptr inreg %sbase
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
@@ -2998,7 +5359,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -3009,7 +5373,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e64 v1, 0xffff, s0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -3022,7 +5391,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
@@ -3033,7 +5405,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -3044,7 +5420,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_undef_hi_immneg128(ptr in
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v0, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
@@ -3063,7 +5442,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -3074,7 +5456,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -3085,7 +5471,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3098,10 +5487,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v0, v[2:3]
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
@@ -3111,7 +5502,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v[2:3]
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -3130,7 +5523,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -3141,7 +5537,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: ; return to shader part epilog
@@ -3152,7 +5552,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3165,10 +5568,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_nop
; GFX1250-NOECC-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX1250-NOECC-SDAG-TRUE16-NEXT: flat_load_d16_hi_b16 v0, v[2:3] offset:-128
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
@@ -3178,7 +5583,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-NOECC-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[2:3], s[2:3], v[0:1]
+; GFX1250-NOECC-SDAG-FAKE16-NEXT: flat_load_d16_hi_b16 v1, v[2:3] offset:-128
; GFX1250-NOECC-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NOECC-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NOECC-SDAG-FAKE16-NEXT: ; return to shader part epilog
@@ -3198,9 +5605,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
@@ -3209,7 +5619,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -3222,11 +5636,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
@@ -3235,9 +5651,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -3254,9 +5673,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
@@ -3265,7 +5687,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -3278,11 +5704,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v0, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
@@ -3291,9 +5719,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_b16 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -3311,9 +5742,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
@@ -3322,7 +5756,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[2:3]
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -3335,11 +5773,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
@@ -3348,9 +5788,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -3368,9 +5811,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
@@ -3379,7 +5825,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-GISEL-NEXT: v_nop
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v[2:3] offset:-128
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -3392,11 +5842,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_u8 v0, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
@@ -3405,9 +5857,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_u8 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -3426,9 +5881,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1]
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
@@ -3437,7 +5895,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3451,11 +5913,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
@@ -3464,7 +5928,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3478,9 +5946,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v1, v0, s[2:3]
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v2, v[0:1]
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -3498,9 +5969,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_nop
; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v2, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
@@ -3509,7 +5983,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3523,11 +6001,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_nop
; GFX1250-SDAG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-SDAG-TRUE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-TRUE16-NEXT: flat_load_i8 v0, v[0:1] offset:-128
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
@@ -3536,7 +6016,11 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3550,9 +6034,12 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-NOECC-NEXT: v_nop
; GFX1250-NOECC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v1, v0, s[2:3] offset:-128
+; GFX1250-NOECC-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NOECC-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NOECC-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NOECC-NEXT: flat_load_d16_hi_i8 v2, v[0:1] offset:-128
; GFX1250-NOECC-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-NOECC-NEXT: v_mov_b32_e32 v0, v2
; GFX1250-NOECC-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 1cf6439589f68..3abdb647c2045 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -7,16 +7,36 @@
; Test using saddr addressing mode of flat_*store_* instructions.
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
-; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i8_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -26,16 +46,36 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr(ptr inreg %sbase, ptr %voff
; Maximum positive offset on gfx10
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
-; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:2047
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:2047
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr_offset_2047:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2 offset:2047
+; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -46,16 +86,36 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_2047(ptr inreg %sbas
; Maximum negative offset on gfx10
define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %sbase, ptr %voffset.ptr, i8 %data) {
-; GFX1250-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_load_b32 v0, v[0:1]
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: flat_store_b8 v0, v2, s[2:3] offset:-2048
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_load_b32 v0, v[0:1]
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:-2048
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i8_zext_vgpr_offset_neg2048:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_b32 v3, v[0:1]
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b8 v[0:1], v2 offset:-2048
+; GFX1250-GISEL-NEXT: s_endpgm
%voffset = load i32, ptr %voffset.ptr
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -78,12 +138,11 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %d
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
@@ -115,12 +174,11 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; GFX1250-SDAG-NEXT: flat_store_b8 v0, v1, s[0:1] offset:-120
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:-120
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
@@ -150,14 +208,30 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
; --------------------------------------------------------------------------------
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, i16 %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i16_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store i16 %data, ptr %gep0
@@ -165,14 +239,30 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i16 %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -181,14 +271,30 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, half %data) {
-; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_f16_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_f16_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store half %data, ptr %gep0
@@ -196,14 +302,30 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, half %data) {
-; GFX1250-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -212,14 +334,30 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i32_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i32_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store i32 %data, ptr %gep0
@@ -227,14 +365,30 @@ define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i32_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -243,14 +397,30 @@ define amdgpu_ps void @flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %voffset, float %data) {
-; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_f32_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_f32_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store float %data, ptr %gep0
@@ -258,14 +428,30 @@ define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, float %data) {
-; GFX1250-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_f32_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -274,14 +460,30 @@ define amdgpu_ps void @flat_store_saddr_f32_zext_vgpr_offset_neg128(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {
-; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_p3_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_p3_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store ptr addrspace(3) %data, ptr %gep0
@@ -289,14 +491,30 @@ define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr(ptr inreg %sbase, i32 %voff
}
define amdgpu_ps void @flat_store_saddr_p3_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, ptr addrspace(3) %data) {
-; GFX1250-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_p3_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -312,7 +530,10 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr:
@@ -322,7 +543,11 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -338,7 +563,10 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i64_zext_vgpr_offset_neg128:
@@ -348,7 +576,11 @@ define amdgpu_ps void @flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -365,7 +597,10 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr:
@@ -375,7 +610,11 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -391,7 +630,10 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f64_zext_vgpr_offset_neg128:
@@ -401,7 +643,11 @@ define amdgpu_ps void @flat_store_saddr_f64_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -418,7 +664,10 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr:
@@ -428,7 +677,11 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -444,7 +697,10 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i32_zext_vgpr_offset_neg128:
@@ -454,7 +710,11 @@ define amdgpu_ps void @flat_store_saddr_v2i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -471,7 +731,10 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr:
@@ -481,7 +744,11 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -497,7 +764,10 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f32_zext_vgpr_offset_neg128:
@@ -507,7 +777,11 @@ define amdgpu_ps void @flat_store_saddr_v2f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -524,7 +798,10 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr:
@@ -534,7 +811,11 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -550,7 +831,10 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i16_zext_vgpr_offset_neg128:
@@ -560,7 +844,11 @@ define amdgpu_ps void @flat_store_saddr_v4i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -577,7 +865,10 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr:
@@ -587,7 +878,11 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -603,7 +898,10 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f16_zext_vgpr_offset_neg128:
@@ -613,7 +911,11 @@ define amdgpu_ps void @flat_store_saddr_v4f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -630,7 +932,10 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voff
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr:
@@ -640,7 +945,11 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr(ptr inreg %sbase, i32 %voff
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -656,7 +965,10 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sb
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_p1_zext_vgpr_offset_neg128:
@@ -666,7 +978,11 @@ define amdgpu_ps void @flat_store_saddr_p1_zext_vgpr_offset_neg128(ptr inreg %sb
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -683,8 +999,10 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr:
@@ -695,7 +1013,11 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -711,8 +1033,10 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3i32_zext_vgpr_offset_neg128:
@@ -723,7 +1047,11 @@ define amdgpu_ps void @flat_store_saddr_v3i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -740,8 +1068,10 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr:
@@ -752,7 +1082,11 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -768,8 +1102,10 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v3f32_zext_vgpr_offset_neg128:
@@ -780,7 +1116,11 @@ define amdgpu_ps void @flat_store_saddr_v3f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -797,8 +1137,10 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr:
@@ -809,7 +1151,11 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -825,8 +1171,10 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6i16_zext_vgpr_offset_neg128:
@@ -837,7 +1185,11 @@ define amdgpu_ps void @flat_store_saddr_v6i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -854,8 +1206,10 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3]
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr:
@@ -866,7 +1220,11 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -882,8 +1240,10 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b96 v0, v[2:4], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b96 v[0:1], v[2:4] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v6f16_zext_vgpr_offset_neg128:
@@ -894,7 +1254,11 @@ define amdgpu_ps void @flat_store_saddr_v6f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v6, v3
-; GFX1250-GISEL-NEXT: flat_store_b96 v0, v[4:6], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b96 v[0:1], v[4:6] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -912,7 +1276,10 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr:
@@ -923,7 +1290,11 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -940,7 +1311,10 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4i32_zext_vgpr_offset_neg128:
@@ -951,7 +1325,11 @@ define amdgpu_ps void @flat_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -969,7 +1347,10 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr:
@@ -980,7 +1361,11 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -997,7 +1382,10 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4f32_zext_vgpr_offset_neg128:
@@ -1008,7 +1396,11 @@ define amdgpu_ps void @flat_store_saddr_v4f32_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1026,7 +1418,10 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr:
@@ -1037,7 +1432,11 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1054,7 +1453,10 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2i64_zext_vgpr_offset_neg128:
@@ -1065,7 +1467,11 @@ define amdgpu_ps void @flat_store_saddr_v2i64_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1083,7 +1489,10 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr:
@@ -1094,7 +1503,11 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1111,7 +1524,10 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2f64_zext_vgpr_offset_neg128:
@@ -1122,7 +1538,11 @@ define amdgpu_ps void @flat_store_saddr_v2f64_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1140,7 +1560,10 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr:
@@ -1151,7 +1574,11 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1168,7 +1595,10 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8i16_zext_vgpr_offset_neg128:
@@ -1179,7 +1609,11 @@ define amdgpu_ps void @flat_store_saddr_v8i16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1197,7 +1631,10 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr:
@@ -1208,7 +1645,11 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr(ptr inreg %sbase, i32 %v
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1225,7 +1666,10 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v8f16_zext_vgpr_offset_neg128:
@@ -1236,7 +1680,11 @@ define amdgpu_ps void @flat_store_saddr_v8f16_zext_vgpr_offset_neg128(ptr inreg
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1254,7 +1702,10 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr:
@@ -1265,7 +1716,11 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1282,7 +1737,10 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v2p1_zext_vgpr_offset_neg128:
@@ -1293,7 +1751,11 @@ define amdgpu_ps void @flat_store_saddr_v2p1_zext_vgpr_offset_neg128(ptr inreg %
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1311,7 +1773,10 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %vo
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3]
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5]
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr:
@@ -1322,7 +1787,11 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr(ptr inreg %sbase, i32 %vo
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3]
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9]
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1339,7 +1808,10 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: flat_store_b128 v0, v[2:5], s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_b128 v[0:1], v[2:5] offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_v4p3_zext_vgpr_offset_neg128:
@@ -1350,7 +1822,11 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-GISEL-NEXT: flat_store_b128 v0, v[6:9], s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_b128 v[0:1], v[6:9] offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1364,16 +1840,34 @@ define amdgpu_ps void @flat_store_saddr_v4p3_zext_vgpr_offset_neg128(ptr inreg %
; --------------------------------------------------------------------------------
define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i32_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i32_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store atomic i32 %data, ptr %gep0 seq_cst, align 4
@@ -1381,16 +1875,34 @@ define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr(ptr inreg %sbase, i
}
define amdgpu_ps void @atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
-; GFX1250-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-NEXT: s_wait_storecnt 0x0
-; GFX1250-NEXT: flat_store_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_SYS
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
+; GFX1250-SDAG-NEXT: flat_store_b32 v[0:1], v2 offset:-128 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i32_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
+; GFX1250-GISEL-NEXT: flat_store_b32 v[2:3], v1 offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1406,9 +1918,12 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr:
@@ -1418,9 +1933,13 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr(ptr inreg %sbase, i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1436,9 +1955,12 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr i
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-SDAG-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
-; GFX1250-SDAG-NEXT: flat_store_b64 v0, v[2:3], s[2:3] offset:-128 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: flat_store_b64 v[0:1], v[2:3] offset:-128 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128:
@@ -1448,9 +1970,13 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr i
; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_wait_storecnt 0x0
-; GFX1250-GISEL-NEXT: flat_store_b64 v0, v[4:5], s[2:3] offset:-128 scope:SCOPE_SYS
+; GFX1250-GISEL-NEXT: flat_store_b64 v[0:1], v[4:5] offset:-128 scope:SCOPE_SYS
; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1470,7 +1996,10 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v[0:1], v2
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
@@ -1479,7 +2008,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[2:3], v1
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
@@ -1488,8 +2021,12 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3]
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-REAL16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v[2:3], v0
; GFX1250-GISEL-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1505,7 +2042,10 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-SDAG-NEXT: v_nop
; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v[0:1], v2 offset:-128
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
@@ -1514,7 +2054,11 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v[2:3], v1 offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
@@ -1523,8 +2067,12 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-REAL16-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v[2:3], v0 offset:-128
; GFX1250-GISEL-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -1535,14 +2083,30 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
}
define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b8 v[0:1], v2
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_d16_hi_b8 v[2:3], v1
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%data.hi = extractelement <2 x i16> %data, i32 1
@@ -1552,14 +2116,30 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_d16_hi_b8 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b8 v[0:1], v2 offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: flat_store_d16_hi_b8 v[2:3], v1 offset:-128
+; GFX1250-GISEL-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1569,5 +2149,6 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg12
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1250: {{.*}}
; GFX1250-SDAG-FAKE16: {{.*}}
; GFX1250-SDAG-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
index 2495fa8a43b29..ec1c395f306a6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.load.b128.ll
@@ -12275,7 +12275,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12286,7 +12289,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12322,7 +12328,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12333,7 +12343,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -12381,7 +12395,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095(ptr addrspace(1)
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12392,7 +12409,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095(ptr addrspace(1)
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12434,7 +12454,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095(ptr addrspace(1)
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12445,7 +12469,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095(ptr addrspace(1)
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -12510,7 +12538,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4096
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4096
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12521,7 +12552,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4096
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4096
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12583,7 +12617,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4096
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4096
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12594,7 +12632,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4096(ptr addrspace(1)
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4096
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4096
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -12643,7 +12685,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4096(ptr addrspace(
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4096
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4096
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12654,7 +12699,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4096(ptr addrspace(
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4096 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4096 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12696,7 +12744,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4096(ptr addrspace(
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4096
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4096
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12707,7 +12759,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4096(ptr addrspace(
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4096 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4096 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -12772,7 +12828,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4097 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4097 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12783,7 +12842,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4097 scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4097 scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12845,7 +12907,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4097 scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4097 scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12856,7 +12922,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg4097(ptr addrspace(
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-4097 scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-4097 scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -12901,7 +12971,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1)
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2047 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2047 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12912,7 +12985,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1)
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2047 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2047 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12948,7 +13024,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1)
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2047 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2047 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -12959,7 +13039,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1)
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2047 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2047 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13008,7 +13092,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2048(ptr addrspace(1)
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2048
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2048
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13019,7 +13106,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2048(ptr addrspace(1)
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2048
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2048
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13061,7 +13151,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2048(ptr addrspace(1)
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2048
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2048
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13072,7 +13166,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_2048(ptr addrspace(1)
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:2048
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:2048
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13117,7 +13215,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2048
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2048
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13128,7 +13229,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2048 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2048 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13164,7 +13268,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2048
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2048
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13175,7 +13283,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2048 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2048 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13224,7 +13336,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2049(ptr addrspace(
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2049 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2049 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13235,7 +13350,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2049(ptr addrspace(
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2049 scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2049 scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13277,7 +13395,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2049(ptr addrspace(
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2049 scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2049 scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13288,7 +13410,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_neg2049(ptr addrspace(
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-2049 scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-2049 scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13353,7 +13479,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:8388607 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:8388607 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13364,7 +13493,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:8388607 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:8388607 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13426,7 +13558,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:8388607 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:8388607 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13437,7 +13573,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0x7FFFFF(ptr addrspace
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:8388607 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:8388607 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -13501,7 +13641,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-8388608
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-8388608
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13512,7 +13655,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-8388608
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-8388608
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13574,7 +13720,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-8388608
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-8388608
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13585,7 +13735,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_0xFFFFFF(ptr addrspace
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-8388608
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:-8388608
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -13634,7 +13788,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095_gep_order(ptr add
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13645,7 +13802,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095_gep_order(ptr add
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13687,7 +13847,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095_gep_order(ptr add
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13698,7 +13862,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_offset_4095_gep_order(ptr add
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:4095 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:4095 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13743,7 +13911,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint(ptr addrspace(1) inr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13754,7 +13925,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint(ptr addrspace(1) inr
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13790,7 +13964,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint(ptr addrspace(1) inr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13801,7 +13979,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint(ptr addrspace(1) inr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13847,7 +14029,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr addr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13858,7 +14043,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr addr
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, v0, s0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, s1, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13894,7 +14082,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr addr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13905,7 +14097,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add(ptr addr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -13951,7 +14147,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13962,7 +14161,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, v0, s0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, 0, s1, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -13998,7 +14200,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -14009,7 +14215,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -14056,7 +14266,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -14067,7 +14280,10 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -14103,7 +14319,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -14114,7 +14334,11 @@ define <4 x float> @global_load_saddr_i8_zext_vgpr_ptrtoint_commute_add_imm_offs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v2, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
@@ -14169,7 +14393,10 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset(ptr addrspace(1) in
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14181,7 +14408,10 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset(ptr addrspace(1) in
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1310-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14253,7 +14483,9 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset(ptr addrspace(1) in
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
@@ -14273,7 +14505,9 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset(ptr addrspace(1) in
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v0
@@ -14329,7 +14563,10 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset_immoffset(ptr addrs
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-24 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14341,7 +14578,10 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset_immoffset(ptr addrs
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1310-SDAG-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-24 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14413,7 +14653,9 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset_immoffset(ptr addrs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-24 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
@@ -14433,7 +14675,9 @@ define <4 x float> @global_load_saddr_i8_zext_uniform_offset_immoffset(ptr addrs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:-24 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v0
@@ -14490,7 +14734,10 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr addr
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14502,7 +14749,10 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr addr
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1310-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14574,7 +14824,9 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr addr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
@@ -14594,7 +14846,9 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add(ptr addr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v0
@@ -14652,7 +14906,10 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offs
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14664,7 +14921,10 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offs
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-SDAG-NEXT: s_mov_b32 s3, 0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1310-SDAG-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
@@ -14736,7 +14996,9 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
@@ -14756,7 +15018,9 @@ define <4 x float> @global_load_saddr_i8_zext_sgpr_ptrtoint_commute_add_imm_offs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s0, s2
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s1, 0
; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v0
@@ -15317,8 +15581,12 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15331,7 +15599,10 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15375,9 +15646,14 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15388,9 +15664,13 @@ define <4 x float> @global_load_saddr_f32_natural_addressing_immoffset(ptr addrs
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:128 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:128 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
@@ -15451,8 +15731,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15466,7 +15748,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15514,11 +15799,15 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15529,10 +15818,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range(ptr addrspace(1) inreg
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !4, !noundef !{}
@@ -15592,8 +15885,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:400 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15607,7 +15902,10 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:400 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15655,11 +15953,15 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
-; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: global_load_b32 v2, v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:400 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
@@ -15670,10 +15972,14 @@ define <4 x float> @global_load_f32_saddr_zext_vgpr_range_imm_offset(ptr addrspa
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v2, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v0, s[0:1] offset:400 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v2
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_load_b128 v[0:3], v[0:1], off offset:400 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr, !range !4, !noundef !{}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index 98dfacb479d73..247db0ac3d621 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -1026,8 +1026,11 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1039,7 +1042,10 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1080,10 +1086,14 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr:
@@ -1093,9 +1103,13 @@ define void @global_store_saddr_i8_zext_vgpr(ptr addrspace(1) inreg %sbase, ptr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1]
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -1142,7 +1156,10 @@ define void @global_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr addrspace(1) i
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
-; GFX1250-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:-128
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-128
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_v4i32_zext_vgpr_offset_neg128:
@@ -1152,7 +1169,10 @@ define void @global_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr addrspace(1) i
; GFX1310-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX1310-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: global_store_b128 v0, v[1:4], s[0:1] offset:-128 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v5, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v6, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[5:6], v[1:4], off offset:-128 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_v4i32_zext_vgpr_offset_neg128:
@@ -1191,7 +1211,11 @@ define void @global_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr addrspace(1) i
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-ISEL-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v7, v2
; GFX1250-ISEL-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v9, v4
-; GFX1250-ISEL-NEXT: global_store_b128 v0, v[6:9], s[0:1] offset:-128
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:-128
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_v4i32_zext_vgpr_offset_neg128:
@@ -1201,7 +1225,11 @@ define void @global_store_saddr_v4i32_zext_vgpr_offset_neg128(ptr addrspace(1) i
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_store_b128 v0, v[1:4], s[0:1] offset:-128 scope:SCOPE_SE
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v5, s0
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v5, vcc_lo, v5, v0
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v6, null, 0, v6, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[5:6], v[1:4], off offset:-128 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
@@ -1252,8 +1280,11 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:2047 scope:SCOPE_DEV
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1265,7 +1296,10 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:2047 scope:SCOPE_DEV
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1306,10 +1340,14 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:2047 scope:SCOPE_DEV
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_2047:
@@ -1319,9 +1357,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_2047(ptr addrspace(1) inreg
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:2047 scope:SCOPE_DEV
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:2047 scope:SCOPE_DEV
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -1373,8 +1415,11 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:-2048 scope:SCOPE_SYS
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-SDAG-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1386,7 +1431,10 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1310-SDAG-NEXT: global_load_b32 v0, v[0:1], off
; GFX1310-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1310-SDAG-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:-2048 scope:SCOPE_SYS
+; GFX1310-SDAG-NEXT: v_add_co_u32 v0, s0, s0, v0
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
+; GFX1310-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1310-SDAG-NEXT: s_set_pc_i64 s[30:31]
;
; GFX906-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1427,10 +1475,14 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1250-ISEL: ; %bb.0:
; GFX1250-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1250-ISEL-NEXT: global_load_b32 v6, v[0:1], off
; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1250-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:-2048 scope:SCOPE_SYS
+; GFX1250-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1250-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1250-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1250-ISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-ISEL-LABEL: global_store_saddr_i8_zext_vgpr_offset_neg2048:
@@ -1440,9 +1492,13 @@ define void @global_store_saddr_i8_zext_vgpr_offset_neg2048(ptr addrspace(1) inr
; GFX1310-ISEL-NEXT: s_wait_samplecnt 0x0
; GFX1310-ISEL-NEXT: s_wait_bvhcnt 0x0
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX1310-ISEL-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: s_wait_loadcnt 0x0
-; GFX1310-ISEL-NEXT: global_store_b128 v0, v[2:5], s[0:1] offset:-2048 scope:SCOPE_SYS
+; GFX1310-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1310-ISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v6
+; GFX1310-ISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1310-ISEL-NEXT: global_store_b128 v[0:1], v[2:5], off offset:-2048 scope:SCOPE_SYS
; GFX1310-ISEL-NEXT: s_set_pc_i64 s[30:31]
%voffset = load i32, ptr addrspace(1) %voffset.ptr
%zext.offset = zext i32 %voffset to i64
@@ -1538,15 +1594,14 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s7, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[0:1], v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v6, s6
-; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[6:7], v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1310-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
@@ -1555,15 +1610,15 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
; GFX1310-SDAG-NEXT: s_clause 0x1
; GFX1310-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1310-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1310-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1310-SDAG-NEXT: ds_load_b64 v[0:1], v0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v0, s0
-; GFX1310-SDAG-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX1310-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1310-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1310-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1310-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1310-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
+; GFX1310-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, s6
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX1310-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-SDAG-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX1310-SDAG-NEXT: s_endpgm
;
; GFX906-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
@@ -1639,35 +1694,37 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
; GFX1250-ISEL-NEXT: v_nop
; GFX1250-ISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX1250-ISEL-NEXT: s_clause 0x1
; GFX1250-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-ISEL-NEXT: ds_load_b64 v[4:5], v0
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: s_load_b32 s4, s[4:5], 0x24 nv
+; GFX1250-ISEL-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
+; GFX1250-ISEL-NEXT: ds_load_b64 v[0:1], v4
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
-; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s4, s6
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s5, 0
; GFX1250-ISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-ISEL-NEXT: s_endpgm
;
; GFX1310-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1310-ISEL: ; %bb.0:
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX1310-ISEL-NEXT: s_clause 0x1
; GFX1310-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX1310-ISEL-NEXT: s_load_b32 s4, s[4:5], 0x24 nv
-; GFX1310-ISEL-NEXT: ds_load_b64 v[4:5], v0
+; GFX1310-ISEL-NEXT: ds_load_b64 v[0:1], v4
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1310-ISEL-NEXT: s_wait_dscnt 0x0
-; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s5, s4
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s6, 0
; GFX1310-ISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1310-ISEL-NEXT: s_endpgm
%sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds
@@ -1757,15 +1814,14 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1250-SDAG-NEXT: s_mov_b32 s7, 0
+; GFX1250-SDAG-NEXT: ds_load_b64 v[0:1], v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v6, s6
-; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1] offset:-120
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[4:5], s[6:7], v[0:1]
+; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: global_store_b128 v[4:5], v[0:3], off offset:-120
; GFX1250-SDAG-NEXT: s_endpgm
;
; GFX1310-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
@@ -1774,15 +1830,15 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
; GFX1310-SDAG-NEXT: s_clause 0x1
; GFX1310-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1310-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1310-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1310-SDAG-NEXT: ds_load_b64 v[0:1], v0
; GFX1310-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1310-SDAG-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v0, s0
-; GFX1310-SDAG-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
-; GFX1310-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX1310-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1310-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1310-SDAG-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1310-SDAG-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1310-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1] offset:-120 scope:SCOPE_SE
+; GFX1310-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, s6
+; GFX1310-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1310-SDAG-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
+; GFX1310-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-SDAG-NEXT: global_store_b128 v[4:5], v[0:3], off offset:-120 scope:SCOPE_SE
; GFX1310-SDAG-NEXT: s_endpgm
;
; GFX906-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
@@ -1858,35 +1914,37 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
; GFX1250-ISEL-NEXT: v_nop
; GFX1250-ISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-ISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-ISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX1250-ISEL-NEXT: s_clause 0x1
; GFX1250-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-ISEL-NEXT: ds_load_b64 v[4:5], v0
-; GFX1250-ISEL-NEXT: s_wait_xcnt 0x0
-; GFX1250-ISEL-NEXT: s_load_b32 s4, s[4:5], 0x24 nv
+; GFX1250-ISEL-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
+; GFX1250-ISEL-NEXT: ds_load_b64 v[0:1], v4
; GFX1250-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-ISEL-NEXT: s_wait_dscnt 0x0
-; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1250-ISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-ISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX1250-ISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-ISEL-NEXT: s_add_co_u32 s0, s4, s6
+; GFX1250-ISEL-NEXT: s_add_co_ci_u32 s1, s5, 0
; GFX1250-ISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1] offset:-120
; GFX1250-ISEL-NEXT: s_endpgm
;
; GFX1310-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1310-ISEL: ; %bb.0:
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX1310-ISEL-NEXT: s_clause 0x1
; GFX1310-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GFX1310-ISEL-NEXT: s_load_b32 s4, s[4:5], 0x24 nv
-; GFX1310-ISEL-NEXT: ds_load_b64 v[4:5], v0
+; GFX1310-ISEL-NEXT: ds_load_b64 v[0:1], v4
; GFX1310-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX1310-ISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1310-ISEL-NEXT: s_wait_dscnt 0x0
-; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s0, v4
-; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1310-ISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1310-ISEL-NEXT: v_readfirstlane_b32 s6, v1
+; GFX1310-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1310-ISEL-NEXT: s_add_co_u32 s0, s5, s4
+; GFX1310-ISEL-NEXT: s_add_co_ci_u32 s1, s6, 0
; GFX1310-ISEL-NEXT: global_store_b128 v4, v[0:3], s[0:1] offset:-120 scope:SCOPE_SE
; GFX1310-ISEL-NEXT: s_endpgm
%sbase = load ptr addrspace(1), ptr addrspace(3) @ptr.in.lds
diff --git a/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll b/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
index 93cf234963b7d..311202164951b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-saddr-offset-imm.ll
@@ -44,15 +44,30 @@ define amdgpu_ps <2 x float> @global_load_scale_add_foldable_nowrap(ptr addrspac
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: global_load_scale_add_foldable_nowrap:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3] offset:128 scale_offset
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: global_load_scale_add_foldable_nowrap:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v0, s[2:3] offset:128 scale_offset
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: global_load_scale_add_foldable_nowrap:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off offset:128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%mul = shl nsw nuw i32 %v, 3
%add = add nsw nuw i32 %mul, 128
%zext.offset = zext i32 %add to i64
@@ -70,16 +85,34 @@ define amdgpu_ps <2 x float> @global_load_scale_add_unfoldable(ptr addrspace(1)
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: global_load_scale_add_unfoldable:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_lshl_add_u32 v0, v0, 3, 0x80
-; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-SDAG-LABEL: global_load_scale_add_unfoldable:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-NEXT: v_lshl_add_u32 v0, v0, 3, 0x80
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-LABEL: global_load_scale_add_unfoldable:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: v_lshl_add_u32 v0, v0, 3, 0x80
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v2, v0
+; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v3, vcc_lo
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
%mul = shl i32 %v, 3
%add = add i32 %mul, 128
%zext.offset = zext i32 %add to i64
@@ -92,5 +125,3 @@ declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
-; GFX1250-GISEL: {{.*}}
-; GFX1250-SDAG: {{.*}}
More information about the llvm-commits
mailing list