[llvm] 20e7308 - [AMDGPU] Port Atomic-Optimizer to use Wave Reduction Intrinsics (#211757)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 21:25:35 PDT 2026
Author: Aaditya
Date: 2026-09-23T09:55:22+05:30
New Revision: 20e7308dbc1b1a6e532a139132e20112ca482b84
URL: https://github.com/llvm/llvm-project/commit/20e7308dbc1b1a6e532a139132e20112ca482b84
DIFF: https://github.com/llvm/llvm-project/commit/20e7308dbc1b1a6e532a139132e20112ca482b84.diff
LOG: [AMDGPU] Port Atomic-Optimizer to use Wave Reduction Intrinsics (#211757)
Currently the atomic optimizer creates reductions via intrinsics,
and introduces new control flows.
Replace this sub-target dependent logic with the existing
wave reduction intrinsics, which get lowered in the backend.
This patch ports the uniform-value and divergent-no-return-value cases.
To port the divergent-with-return-value cases, additional scan intrinsics
will need to be added.
Added:
Modified:
llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll
llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 18c7cac7df885..aa822c13dd8fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -652,6 +652,38 @@ static Constant *getIdentityValueForAtomicOp(Type *const Ty,
}
}
+static Intrinsic::ID getWaveReductionIntrinsic(AtomicRMWInst::BinOp Op) {
+ switch (Op) {
+ default:
+ llvm_unreachable(
+ "Atomic Op yet to be ported to use Wave Reduction intrinsics.");
+ case AtomicRMWInst::Add:
+ case AtomicRMWInst::Sub:
+ return Intrinsic::amdgcn_wave_reduce_add;
+ case AtomicRMWInst::FAdd:
+ case AtomicRMWInst::FSub:
+ return Intrinsic::amdgcn_wave_reduce_fadd;
+ case AtomicRMWInst::And:
+ return Intrinsic::amdgcn_wave_reduce_and;
+ case AtomicRMWInst::Or:
+ return Intrinsic::amdgcn_wave_reduce_or;
+ case AtomicRMWInst::Xor:
+ return Intrinsic::amdgcn_wave_reduce_xor;
+ case AtomicRMWInst::UMax:
+ return Intrinsic::amdgcn_wave_reduce_umax;
+ case AtomicRMWInst::Max:
+ return Intrinsic::amdgcn_wave_reduce_max;
+ case AtomicRMWInst::FMax:
+ return Intrinsic::amdgcn_wave_reduce_fmax;
+ case AtomicRMWInst::UMin:
+ return Intrinsic::amdgcn_wave_reduce_umin;
+ case AtomicRMWInst::Min:
+ return Intrinsic::amdgcn_wave_reduce_min;
+ case AtomicRMWInst::FMin:
+ return Intrinsic::amdgcn_wave_reduce_fmin;
+ }
+}
+
static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) {
const ConstantInt *CI = dyn_cast<ConstantInt>(LHS);
return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS);
@@ -742,6 +774,8 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
Function *F = I.getFunction();
LLVMContext &C = F->getContext();
+ const bool NeedResult = !I.use_empty();
+ const bool UseWaveReductionIntrinsic = !ValDivergent || !NeedResult;
// For atomic sub, perform scan with add operation and allow one lane to
// subtract the reduced value later.
@@ -756,16 +790,20 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
Value *ExclScan = nullptr;
Value *NewV = nullptr;
- const bool NeedResult = !I.use_empty();
-
BasicBlock *ComputeLoop = nullptr;
BasicBlock *ComputeEnd = nullptr;
- // If we have a divergent value in each lane, we need to combine the value
- // using DPP.
- if (ValDivergent) {
+ if (UseWaveReductionIntrinsic) {
+ // Build reductions with wave-reduce intrinsics.
+ unsigned Strategy = ScanImpl == ScanOptions::DPP ? 2 : 1;
+ Intrinsic::ID WaveRedIntrinsic = getWaveReductionIntrinsic(Op);
+ NewV = B.CreateIntrinsic(WaveRedIntrinsic, Ty, {V, B.getInt32(Strategy)});
+ } else {
+ // If we have a divergent value in each lane, we need to combine the value
+ // using DPP.
+ assert(ValDivergent && NeedResult);
if (ScanImpl == ScanOptions::DPP) {
- // First we need to set all inactive invocations to the identity value, so
- // that they can correctly contribute to the final result.
+ // First we need to set all inactive invocations to the identity value,
+ // so that they can correctly contribute to the final result.
NewV =
B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
if (!NeedResult && ST.hasPermlane16Insts()) {
@@ -795,49 +833,6 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
} else {
llvm_unreachable("Atomic Optimzer is disabled for None strategy");
}
- } else {
- switch (Op) {
- default:
- llvm_unreachable("Unhandled atomic op");
-
- case AtomicRMWInst::Add:
- case AtomicRMWInst::Sub: {
- // The new value we will be contributing to the atomic operation is the
- // old value times the number of active lanes.
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
- NewV = buildMul(B, V, Ctpop);
- break;
- }
- case AtomicRMWInst::FAdd:
- case AtomicRMWInst::FSub: {
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Int32Ty, false);
- Value *const CtpopFP = B.CreateUIToFP(Ctpop, Ty);
- NewV = B.CreateFMul(V, CtpopFP);
- break;
- }
- case AtomicRMWInst::And:
- case AtomicRMWInst::Or:
- case AtomicRMWInst::Max:
- case AtomicRMWInst::Min:
- case AtomicRMWInst::UMax:
- case AtomicRMWInst::UMin:
- case AtomicRMWInst::FMin:
- case AtomicRMWInst::FMax:
- // These operations with a uniform value are idempotent: doing the atomic
- // operation multiple times has the same effect as doing it once.
- NewV = V;
- break;
-
- case AtomicRMWInst::Xor:
- // The new value we will be contributing to the atomic operation is the
- // old value times the parity of the number of active lanes.
- Value *const Ctpop = B.CreateIntCast(
- B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot), Ty, false);
- NewV = buildMul(B, V, B.CreateAnd(Ctpop, 1));
- break;
- }
}
// We only want a single lane to enter our new control flow, and we do this
@@ -865,7 +860,7 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
// ComputeEnd block. We also need to set up predecessor to next block when
// single lane done updating the final reduced value.
BasicBlock *Predecessor = nullptr;
- if (ValDivergent && ScanImpl == ScanOptions::Iterative) {
+ if (NeedResult && ValDivergent && ScanImpl == ScanOptions::Iterative) {
// Move terminator from I's block to ComputeEnd block.
//
// OriginalBB is known to have a branch as terminator because
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index 2c07b8683bef3..a8942ab52166c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -1586,16 +1586,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB36_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1609,16 +1612,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB36_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1635,22 +1641,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat(ptr addrspace(1) %pt
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB36_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1665,16 +1675,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB37_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -1686,16 +1699,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB37_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1712,22 +1728,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent(ptr addrspace(
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB37_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1742,16 +1762,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB38_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1765,16 +1788,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_system:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB38_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1791,22 +1817,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_system(ptr addrspace
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB38_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -1821,16 +1851,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB39_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -1842,16 +1875,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB39_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1868,22 +1904,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_flush(ptr addrspace(
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB39_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -2024,16 +2064,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX90A-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB43_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3]
@@ -2045,16 +2088,19 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX942-LABEL: global_atomic_fadd_f64_noret_pat_agent_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB43_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2071,22 +2117,26 @@ define amdgpu_kernel void @global_atomic_fadd_f64_noret_pat_agent_safe(ptr addrs
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB43_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT: global_atomic_add_f64 v2, v[0:1], s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: global_inv scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -2427,16 +2477,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB51_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2447,16 +2500,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB51_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2471,18 +2527,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB51_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB51_2:
@@ -2496,16 +2557,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB52_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2516,16 +2580,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB52_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2540,18 +2607,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB52_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB52_2:
@@ -2565,16 +2637,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX90A-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX90A-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB53_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX90A-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: ds_add_f64 v2, v[0:1]
@@ -2585,16 +2660,19 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX942-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX942: ; %bb.0: ; %main_body
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX942-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v0
+; GFX942-NEXT: v_readfirstlane_b32 s1, v1
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB53_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_load_dword s2, s[4:5], 0x24
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX942-NEXT: v_mul_f64 v[0:1], v[0:1], 4.0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s2
; GFX942-NEXT: ds_add_f64 v2, v[0:1]
@@ -2609,18 +2687,23 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_mov_b32 s0, exec_lo
-; GFX1250-NEXT: s_mov_b32 s1, exec_lo
-; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1250-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT: s_mov_b32 s2, exec_lo
+; GFX1250-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1250-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1250-NEXT: s_cbranch_execz .LBB53_2
; GFX1250-NEXT: ; %bb.1:
-; GFX1250-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
; GFX1250-NEXT: ds_add_f64 v2, v[0:1]
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: .LBB53_2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
index 492b7799e0e5f..cb6076191cbc9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-atomic-fadd.f32-no-rtn.ll
@@ -80,50 +80,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX908-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX908-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX908-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX908-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX908-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX908-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX908-NEXT: early-clobber %33:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX908-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX908-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX908-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX908-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX908-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX908-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: S_BRANCH %bb.3
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.3 (%ir-block.31):
+ ; GFX908-NEXT: bb.3 (%ir-block.17):
; GFX908-NEXT: successors: %bb.4(0x80000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %33
; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.4.Flow:
; GFX908-NEXT: successors: %bb.5(0x80000000)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.5 (%ir-block.33):
+ ; GFX908-NEXT: bb.5 (%ir-block.19):
; GFX908-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX908-NEXT: S_ENDPGM 0
;
@@ -153,50 +129,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX90A-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX90A-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX90A-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX90A-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX90A-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX90A-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX90A-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX90A-NEXT: early-clobber %33:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX90A-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX90A-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX90A-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX90A-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX90A-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.3 (%ir-block.31):
+ ; GFX90A-NEXT: bb.3 (%ir-block.17):
; GFX90A-NEXT: successors: %bb.4(0x80000000)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %33
; GFX90A-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX90A-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.4.Flow:
; GFX90A-NEXT: successors: %bb.5(0x80000000)
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.5 (%ir-block.33):
+ ; GFX90A-NEXT: bb.5 (%ir-block.19):
; GFX90A-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_ENDPGM 0
;
@@ -226,50 +178,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX942-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY7]], [[COPY8]], implicit $exec
; GFX942-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
; GFX942-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 [[COPY9]], [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX942-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX942-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX942-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY10]], [[DEF]], implicit $exec
- ; GFX942-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY11]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY12]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY13]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY14]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY15]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX942-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY16]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX942-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX942-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX942-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_ADD_F32_e64_5]], [[S_MOV_B32_2]]
- ; GFX942-NEXT: early-clobber %32:sreg_32 = STRICT_WWM [[V_READLANE_B32_]], implicit $exec
- ; GFX942-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY17]], implicit $exec
- ; GFX942-NEXT: [[COPY18:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY18]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX942-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX942-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX942-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_MBCNT_HI_U32_B32_e64_]], [[COPY11]], implicit $exec
+ ; GFX942-NEXT: [[COPY12:%[0-9]+]]:sreg_64_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64_xexec = SI_IF [[COPY12]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: S_BRANCH %bb.3
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.3 (%ir-block.31):
+ ; GFX942-NEXT: bb.3 (%ir-block.17):
; GFX942-NEXT: successors: %bb.4(0x80000000)
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY %32
; GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY19]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY10]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: bb.4.Flow:
; GFX942-NEXT: successors: %bb.5(0x80000000)
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: bb.5 (%ir-block.33):
+ ; GFX942-NEXT: bb.5 (%ir-block.19):
; GFX942-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX942-NEXT: S_ENDPGM 0
;
@@ -282,7 +210,6 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX11-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
; GFX11-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_32_xm0_xexec = SI_PS_LIVE
; GFX11-NEXT: [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[SI_PS_LIVE]], %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.2
@@ -295,44 +222,26 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY3]]
; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX11-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[COPY5]], implicit $exec
- ; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2147483648
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[DEF1:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GFX11-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY2]], 0, [[COPY6]], [[DEF1]], implicit $exec
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY7]], [[V_SET_INACTIVE_B32_]], 353, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY8]], [[V_ADD_F32_e64_]], 354, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY9]], [[V_ADD_F32_e64_1]], 356, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
- ; GFX11-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY10]], [[V_ADD_F32_e64_2]], 360, 15, 15, 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
- ; GFX11-NEXT: [[V_PERMLANEX16_B32_e64_:%[0-9]+]]:vgpr_32 = V_PERMLANEX16_B32_e64 0, [[V_ADD_F32_e64_3]], 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], [[COPY11]], 0, implicit $exec
- ; GFX11-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, [[V_PERMLANEX16_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; GFX11-NEXT: early-clobber %24:vgpr_32 = STRICT_WWM [[V_ADD_F32_e64_4]], implicit $exec
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[V_MBCNT_LO_U32_B32_e64_]], [[COPY12]], implicit $exec
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY13]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX11-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY2]], 2, implicit $exec
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 [[V_MBCNT_LO_U32_B32_e64_]], [[COPY7]], implicit $exec
+ ; GFX11-NEXT: [[COPY8:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec = SI_IF [[COPY8]], %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.3
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: bb.3 (%ir-block.24):
+ ; GFX11-NEXT: bb.3 (%ir-block.13):
; GFX11-NEXT: successors: %bb.4(0x80000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], %24, [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
+ ; GFX11-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR [[V_MOV_B32_e32_]], [[COPY6]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (f32) on %ir.ptr, addrspace 1)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: bb.4.Flow:
; GFX11-NEXT: successors: %bb.5(0x80000000)
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: SI_END_CF [[SI_IF1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: bb.5 (%ir-block.26):
+ ; GFX11-NEXT: bb.5 (%ir-block.15):
; GFX11-NEXT: SI_END_CF [[SI_IF]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX11-NEXT: S_ENDPGM 0
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0, !atomic.ignore.denormal.mode !0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
index d2ff3c25a1acb..ee1758316d717 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
@@ -10,13 +10,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i8 @llvm.amdgcn.wave.reduce.or.i8(i8 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[VAL]] monotonic, align 1
+; CHECK-NEXT: [[TMP10:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i8 [[TMP8]] monotonic, align 1
; CHECK-NEXT: br label %[[BB10]]
; CHECK: [[BB10]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i8 [ poison, [[TMP0:%.*]] ], [ [[TMP10]], %[[BB8]] ]
; CHECK-NEXT: [[TMP16:%.*]] = zext i8 [[TMP11]] to i32
; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i8
@@ -40,9 +41,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i8
-; CHECK-NEXT: [[TMP9:%.*]] = mul i8 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i8 @llvm.amdgcn.wave.reduce.add.i8(i8 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
; CHECK: [[BB11]]:
@@ -85,13 +84,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
+; CHECK-NEXT: [[TMP8:%.*]] = call i16 @llvm.amdgcn.wave.reduce.or.i16(i16 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[VAL]] monotonic, align 2
+; CHECK-NEXT: [[TMP10:%.*]] = atomicrmw or ptr addrspace(1) [[UNIFORM_PTR]], i16 [[TMP8]] monotonic, align 2
; CHECK-NEXT: br label %[[BB10]]
; CHECK: [[BB10]]:
-; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP9]], %[[BB8]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi i16 [ poison, [[TMP0:%.*]] ], [ [[TMP10]], %[[BB8]] ]
; CHECK-NEXT: [[TMP16:%.*]] = zext i16 [[TMP11]] to i32
; CHECK-NEXT: [[TMP17:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP16]])
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP17]] to i16
@@ -115,9 +115,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; CHECK-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; CHECK-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i16
-; CHECK-NEXT: [[TMP9:%.*]] = mul i16 [[VAL]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.amdgcn.wave.reduce.add.i16(i16 [[VAL]], i32 1)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; CHECK-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
; CHECK: [[BB11]]:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll
index faa4f1f20bf18..2e51932adc4e3 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-volatile.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=iterative>,verify<domtree>' %s | FileCheck --check-prefix=IR %s
-; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=dpp>,verify<domtree>' %s | FileCheck --check-prefix=IR %s
+; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=iterative>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-ITER %s
+; RUN: opt -S -mtriple=amdgpu9.00-amd-amdhsa -passes='amdgpu-atomic-optimizer<strategy=dpp>,verify<domtree>' %s | FileCheck --check-prefixes=IR,IR-DPP %s
; Folding a wavefront into a single atomic would change the number of volatile
; operations, so these must be left alone.
@@ -53,30 +53,51 @@ entry:
; Negative control: the same atomic without the volatile marker must be folded.
define amdgpu_kernel void @nonvolatile_global(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
-; IR-LABEL: define amdgpu_kernel void @nonvolatile_global(
-; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
-; IR-NEXT: [[ENTRY:.*]]:
-; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
-; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
-; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
-; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 5, [[TMP7]]
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
-; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[INOUT]], i32 [[TMP8]] syncscope("agent") acq_rel, align 4
-; IR-NEXT: br label %[[BB12]]
-; IR: [[BB12]]:
-; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP11]], %[[BB10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = mul i32 5, [[TMP5]]
-; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]]
-; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT]], align 4
-; IR-NEXT: ret void
+; IR-ITER-LABEL: define amdgpu_kernel void @nonvolatile_global(
+; IR-ITER-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
+; IR-ITER-NEXT: [[ENTRY:.*]]:
+; IR-ITER-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITER-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-ITER-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-ITER-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-ITER-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-ITER-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-ITER-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 1)
+; IR-ITER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-ITER-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-ITER: [[BB8]]:
+; IR-ITER-NEXT: [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[INOUT]], i32 [[TMP6]] syncscope("agent") acq_rel, align 4
+; IR-ITER-NEXT: br label %[[BB10]]
+; IR-ITER: [[BB10]]:
+; IR-ITER-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
+; IR-ITER-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
+; IR-ITER-NEXT: [[TMP13:%.*]] = mul i32 5, [[TMP5]]
+; IR-ITER-NEXT: [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
+; IR-ITER-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
+; IR-ITER-NEXT: ret void
+;
+; IR-DPP-LABEL: define amdgpu_kernel void @nonvolatile_global(
+; IR-DPP-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[INOUT:%.*]]) {
+; IR-DPP-NEXT: [[ENTRY:.*]]:
+; IR-DPP-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-DPP-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-DPP-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-DPP-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 2)
+; IR-DPP-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-DPP-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-DPP: [[BB8]]:
+; IR-DPP-NEXT: [[TMP9:%.*]] = atomicrmw add ptr addrspace(1) [[INOUT]], i32 [[TMP6]] syncscope("agent") acq_rel, align 4
+; IR-DPP-NEXT: br label %[[BB10]]
+; IR-DPP: [[BB10]]:
+; IR-DPP-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
+; IR-DPP-NEXT: [[TMP13:%.*]] = mul i32 5, [[TMP5]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
+; IR-DPP-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
+; IR-DPP-NEXT: ret void
;
entry:
%old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
@@ -146,30 +167,51 @@ entry:
; Negative control: only bit 31 means volatile, so a plain cachepolicy bit must
; not block the optimization.
define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
-; IR-LABEL: define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(
-; IR-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
-; IR-NEXT: [[ENTRY:.*]]:
-; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
-; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
-; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
-; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 5, [[TMP7]]
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
-; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[TMP8]], ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 1)
-; IR-NEXT: br label %[[BB12]]
-; IR: [[BB12]]:
-; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP11]], %[[BB10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = mul i32 5, [[TMP5]]
-; IR-NEXT: [[TMP16:%.*]] = add i32 [[TMP14]], [[TMP15]]
-; IR-NEXT: store i32 [[TMP16]], ptr addrspace(1) [[OUT]], align 4
-; IR-NEXT: ret void
+; IR-ITER-LABEL: define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(
+; IR-ITER-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
+; IR-ITER-NEXT: [[ENTRY:.*]]:
+; IR-ITER-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITER-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-ITER-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-ITER-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-ITER-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-ITER-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-ITER-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 1)
+; IR-ITER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-ITER-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-ITER: [[BB8]]:
+; IR-ITER-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[TMP6]], ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 1)
+; IR-ITER-NEXT: br label %[[BB10]]
+; IR-ITER: [[BB10]]:
+; IR-ITER-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
+; IR-ITER-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
+; IR-ITER-NEXT: [[TMP13:%.*]] = mul i32 5, [[TMP5]]
+; IR-ITER-NEXT: [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
+; IR-ITER-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
+; IR-ITER-NEXT: ret void
+;
+; IR-DPP-LABEL: define amdgpu_kernel void @nonvolatile_raw_ptr_buffer(
+; IR-DPP-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(8) [[INOUT:%.*]]) {
+; IR-DPP-NEXT: [[ENTRY:.*]]:
+; IR-DPP-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-DPP-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-DPP-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-DPP-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 5, i32 2)
+; IR-DPP-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-DPP-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR-DPP: [[BB8]]:
+; IR-DPP-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add.i32(i32 [[TMP6]], ptr addrspace(8) [[INOUT]], i32 0, i32 0, i32 1)
+; IR-DPP-NEXT: br label %[[BB10]]
+; IR-DPP: [[BB10]]:
+; IR-DPP-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %[[ENTRY]] ], [ [[TMP9]], %[[BB8]] ]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP11]])
+; IR-DPP-NEXT: [[TMP13:%.*]] = mul i32 5, [[TMP5]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = add i32 [[TMP12]], [[TMP13]]
+; IR-DPP-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT]], align 4
+; IR-DPP-NEXT: ret void
;
entry:
%old = call i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32 5, ptr addrspace(8) %inout, i32 0, i32 0, i32 1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
index 9fafedcaba475..80348a9a79d46 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
@@ -34,30 +34,16 @@ define amdgpu_kernel void @ham(ptr addrspace(4) %arg) {
; CHECK-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; CHECK-NEXT: br label [[COMPUTELOOP:%.*]]
-; CHECK: 7:
-; CHECK-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[ADDRSPACECAST]], i32 [[TMP13:%.*]] syncscope("agent-one-as") monotonic, align 4
-; CHECK-NEXT: br label [[TMP9:%.*]]
-; CHECK: 9:
+; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[PHI]], i32 1)
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT: br i1 [[TMP7]], label [[TMP10:%.*]], label [[TMP9:%.*]]
+; CHECK: 8:
+; CHECK-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[ADDRSPACECAST]], i32 [[TMP13]] syncscope("agent-one-as") monotonic, align 4
+; CHECK-NEXT: br label [[TMP9]]
+; CHECK: 10:
; CHECK-NEXT: br label [[BB8]]
; CHECK: bb8:
; CHECK-NEXT: br label [[BB4]]
-; CHECK: ComputeLoop:
-; CHECK-NEXT: [[ACCUMULATOR:%.*]] = phi i32 [ 0, [[BB7]] ], [ [[TMP13]], [[COMPUTELOOP]] ]
-; CHECK-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP6]], [[BB7]] ], [ [[TMP16:%.*]], [[COMPUTELOOP]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[PHI]], i32 [[TMP11]])
-; CHECK-NEXT: [[TMP13]] = add i32 [[ACCUMULATOR]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = shl i64 1, [[TMP10]]
-; CHECK-NEXT: [[TMP15:%.*]] = xor i64 [[TMP14]], -1
-; CHECK-NEXT: [[TMP16]] = and i64 [[ACTIVEBITS]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 0
-; CHECK-NEXT: br i1 [[TMP17]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; CHECK: ComputeEnd:
-; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP5]], 0
-; CHECK-NEXT: br i1 [[TMP18]], label [[TMP7:%.*]], label [[TMP9]]
;
bb:
%call = tail call i32 @quux()
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index f5fee7177d98d..67c115ac2ed76 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -21,16 +21,16 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -49,16 +49,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -77,16 +77,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -104,16 +104,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -132,15 +132,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -159,19 +159,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -189,18 +190,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -218,20 +219,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -250,19 +251,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -280,19 +280,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -310,18 +311,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -345,26 +346,26 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -375,26 +376,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -405,26 +406,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -434,29 +435,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -464,15 +466,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -492,31 +494,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -524,16 +527,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -553,33 +556,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -587,20 +589,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -617,31 +618,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -649,16 +651,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1773,16 +1775,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -1802,16 +1804,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -1831,16 +1833,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1859,16 +1861,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -1888,15 +1890,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -1916,19 +1918,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1947,18 +1950,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1977,20 +1980,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2010,19 +2013,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2041,19 +2043,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2072,18 +2075,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -2107,26 +2110,26 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB6_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB6_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -2137,26 +2140,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB6_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -2167,26 +2170,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB6_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -2196,27 +2199,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB6_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB6_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2227,15 +2230,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB6_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -2255,28 +2258,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB6_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB6_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2288,16 +2292,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB6_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2318,29 +2322,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB6_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB6_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2353,20 +2357,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB6_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB6_2:
@@ -2385,28 +2388,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB6_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB6_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2418,16 +2422,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB6_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index dc1801f9b549f..f7045fe08693a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -38,22 +38,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-LABEL: add_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s6, s6, 5
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
@@ -70,22 +70,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
+; GFX8-NEXT: s_mul_i32 s6, s6, 5
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s8, s2
; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -102,22 +102,22 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
+; GFX9-NEXT: s_mul_i32 s6, s6, 5
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s8, s2
; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -134,19 +134,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: add_i32_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_mov_b32 s8, s2
; GFX1064-NEXT: s_mov_b32 s9, s3
@@ -168,18 +168,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i32_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_mul_i32 s5, s5, 5
-; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
@@ -201,20 +201,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: add_i32_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_mov_b32 s8, s2
; GFX1164-NEXT: s_mov_b32 s9, s3
@@ -235,19 +236,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i32_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_mul_i32 s5, s5, 5
-; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: v_mov_b32_e32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -268,22 +269,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: add_i32_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB0_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: v_mov_b32_e32 v1, s6
+; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -303,19 +303,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i32_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB0_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_mul_i32 s5, s5, 5
-; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -335,20 +335,21 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: add_i32_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: v_mov_b32_e32 v1, s6
+; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -371,19 +372,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: add_i32_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_mul_i32 s5, s5, 5
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -412,30 +413,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-LABEL: add_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s8, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s8, s2
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
-; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_mul_i32 s7, s6, s7
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: .LBB1_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -446,30 +447,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s8, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX8-NEXT: s_mul_i32 s7, s6, s7
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s8, s2
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -480,30 +481,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s8, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX9-NEXT: s_mul_i32 s7, s6, s7
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s8, s2
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -515,24 +516,24 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s15, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s6, s8, s6
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: v_mov_b32_e32 v1, s6
-; GFX1064-NEXT: s_mov_b32 s12, s2
-; GFX1064-NEXT: s_mov_b32 s13, s3
-; GFX1064-NEXT: buffer_atomic_add v1, off, s[12:15], 0 glc
+; GFX1064-NEXT: s_mul_i32 s7, s6, s7
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b32 s8, s2
+; GFX1064-NEXT: s_mov_b32 s9, s3
+; GFX1064-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
@@ -541,7 +542,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s8, v0, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -552,19 +553,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s7, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s7, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s7
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
; GFX1032-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -586,25 +587,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s15, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s6, s8, s6
-; GFX1164-NEXT: s_mov_b32 s14, -1
-; GFX1164-NEXT: v_mov_b32_e32 v1, s6
-; GFX1164-NEXT: s_mov_b32 s12, s2
-; GFX1164-NEXT: s_mov_b32 s13, s3
-; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], 0 glc
+; GFX1164-NEXT: s_mul_i32 s7, s6, s7
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b32 s8, s2
+; GFX1164-NEXT: s_mov_b32 s9, s3
+; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
@@ -613,7 +615,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s8, v0, s[2:3]
+; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0
@@ -624,20 +626,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s5, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1132-NEXT: s_mov_b32 s5, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s6, s4, s6
-; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: v_mov_b32_e32 v1, s6
+; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -659,27 +661,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB1_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_mul_i32 s6, s8, s6
-; GFX1264-NEXT: s_mov_b32 s14, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: s_mov_b32 s12, s2
-; GFX1264-NEXT: s_mov_b32 s13, s3
-; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: s_mul_i32 s7, s6, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s8, s2
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB1_2:
@@ -687,7 +688,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: v_readfirstlane_b32 s2, v1
; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s8, v0, s[2:3]
+; GFX1264-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null
@@ -698,22 +699,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s5, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1232-NEXT: s_mov_b32 s5, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB1_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-NEXT: s_mul_i32 s6, s4, s6
-; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -734,27 +733,28 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB1_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s15, 0x31016000
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s6, s8, s6
-; GFX1364-NEXT: s_mov_b32 s14, -1
-; GFX1364-NEXT: v_mov_b32_e32 v1, s6
-; GFX1364-NEXT: s_mov_b32 s12, s2
-; GFX1364-NEXT: s_mov_b32 s13, s3
+; GFX1364-NEXT: s_mul_i32 s7, s6, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s8, s2
+; GFX1364-NEXT: s_mov_b32 s9, s3
; GFX1364-NEXT: global_wb scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_storecnt 0x0
-; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
@@ -763,7 +763,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s8, v0, s[2:3]
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: buffer_store_b32 v0, off, s[0:3], null
@@ -774,20 +774,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s5, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1332-NEXT: s_mov_b32 s5, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB1_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s6, s4, s6
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
@@ -1850,24 +1850,28 @@ entry:
define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; GFX7LESS-LABEL: add_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
+; GFX7LESS-NEXT: s_mul_i32 s6, s6, 0
+; GFX7LESS-NEXT: s_add_u32 s13, s7, s6
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s13
; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
@@ -1889,24 +1893,28 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i64_constant:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB3_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
+; GFX8-NEXT: s_mul_i32 s12, s6, 5
+; GFX8-NEXT: s_mul_i32 s6, s6, 0
+; GFX8-NEXT: s_add_u32 s13, s7, s6
; GFX8-NEXT: s_mov_b32 s11, 0xf000
; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s8, s2
; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v1, s13
; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -1928,23 +1936,26 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: add_i64_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB3_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
+; GFX9-NEXT: s_mul_i32 s12, s6, 5
+; GFX9-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX9-NEXT: s_mul_i32 s6, s6, 0
+; GFX9-NEXT: s_add_u32 s13, s7, s6
; GFX9-NEXT: s_mov_b32 s11, 0xf000
; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s8, s2
; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s13
; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1965,19 +1976,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: add_i64_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1064-NEXT: s_mul_i32 s8, s6, 0
; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_mov_b32 s8, s2
@@ -2001,18 +2015,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s5, s5, 5
+; GFX1032-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1032-NEXT: s_mul_i32 s8, s5, 0
+; GFX1032-NEXT: s_mul_i32 s6, s5, 5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
@@ -2036,20 +2053,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: add_i64_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1164-NEXT: s_mul_i32 s8, s6, 0
; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_mov_b32 s8, s2
@@ -2073,19 +2094,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1132-NEXT: s_mul_i32 s8, s5, 0
+; GFX1132-NEXT: s_mul_i32 s6, s5, 5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s5, s5, 5
; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -2108,22 +2133,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: add_i64_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB3_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: v_mov_b32_e32 v1, 0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1264-NEXT: s_mul_i32 s8, s6, 0
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_add_co_u32 s7, s7, s8
; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
@@ -2146,19 +2173,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB3_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1232-NEXT: s_mul_i32 s8, s5, 0
+; GFX1232-NEXT: s_mul_i32 s6, s5, 5
+; GFX1232-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -2180,20 +2211,24 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: add_i64_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB3_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1364-NEXT: s_mul_i32 s8, s6, 0
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
@@ -2219,19 +2254,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: add_i64_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB3_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1332-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1332-NEXT: s_mul_i32 s8, s5, 0
+; GFX1332-NEXT: s_mul_i32 s6, s5, 5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1332-NEXT: s_mul_i32 s5, s5, 5
; GFX1332-NEXT: s_mov_b32 s10, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -2261,40 +2300,42 @@ entry:
define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %additive) {
; GFX7LESS-LABEL: add_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s8, s[0:1]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[8:9], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s8, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s9, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s9, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s4, v0
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: s_mul_i32 s3, s5, s2
-; GFX7LESS-NEXT: s_mul_i32 s2, s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s3, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_mul_i32 s12, s6, s8
+; GFX7LESS-NEXT: s_mul_i32 s8, s7, s8
+; GFX7LESS-NEXT: s_add_u32 s13, s9, s8
+; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s10, -1
+; GFX7LESS-NEXT: s_mov_b32 s8, s2
+; GFX7LESS-NEXT: s_mov_b32 s9, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s13
+; GFX7LESS-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: .LBB4_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v1, s5, v2
-; GFX7LESS-NEXT: v_mul_hi_u32 v3, s4, v2
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mul_lo_u32 v1, s7, v2
+; GFX7LESS-NEXT: v_mul_hi_u32 v3, s6, v2
; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s4, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s5, v0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
@@ -2305,38 +2346,41 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i64_uniform:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s4, v0, 0
-; GFX8-NEXT: s_mul_i32 s2, s5, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s2, v1
-; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX8-NEXT: s_mul_i32 s12, s6, s8
+; GFX8-NEXT: s_mul_i32 s8, s7, s8
+; GFX8-NEXT: s_add_u32 s13, s9, s8
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: s_mov_b32 s8, s2
+; GFX8-NEXT: s_mov_b32 s9, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v1, s13
+; GFX8-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: .LBB4_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mul_lo_u32 v3, s5, v2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s4, v2, v[0:1]
+; GFX8-NEXT: v_mul_lo_u32 v3, s7, v2
+; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v2, v[0:1]
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
@@ -2348,27 +2392,27 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[8:9], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s8, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: s_mul_i32 s3, s7, s2
-; GFX9-NEXT: s_mul_hi_u32 s8, s6, s2
-; GFX9-NEXT: s_add_i32 s8, s8, s3
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[12:15], 0 glc
+; GFX9-NEXT: s_mul_i32 s12, s6, s8
+; GFX9-NEXT: s_mul_hi_u32 s9, s6, s8
+; GFX9-NEXT: s_mul_i32 s8, s7, s8
+; GFX9-NEXT: s_add_u32 s13, s9, s8
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_mov_b32 s8, s2
+; GFX9-NEXT: s_mov_b32 s9, s3
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s13
+; GFX9-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: .LBB4_2:
@@ -2391,23 +2435,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[8:9], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s9, s7, s8
-; GFX1064-NEXT: s_mul_hi_u32 s10, s6, s8
+; GFX1064-NEXT: s_mul_hi_u32 s9, s6, s8
+; GFX1064-NEXT: s_mul_i32 s10, s7, s8
; GFX1064-NEXT: s_mul_i32 s8, s6, s8
-; GFX1064-NEXT: s_add_i32 s10, s10, s9
+; GFX1064-NEXT: s_add_u32 s9, s9, s10
; GFX1064-NEXT: v_mov_b32_e32 v0, s8
-; GFX1064-NEXT: v_mov_b32_e32 v1, s10
+; GFX1064-NEXT: v_mov_b32_e32 v1, s9
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_mov_b32 s10, -1
; GFX1064-NEXT: s_mov_b32 s8, s2
; GFX1064-NEXT: s_mov_b32 s9, s3
@@ -2433,22 +2477,22 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s8, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s8, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s8
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s8, s7, s5
; GFX1032-NEXT: s_mul_hi_u32 s9, s6, s5
-; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_add_i32 s9, s9, s8
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s10, s7, s5
+; GFX1032-NEXT: s_mul_i32 s8, s6, s5
+; GFX1032-NEXT: s_add_u32 s9, s9, s10
+; GFX1032-NEXT: v_mov_b32_e32 v0, s8
; GFX1032-NEXT: v_mov_b32_e32 v1, s9
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_mov_b32 s8, s2
; GFX1032-NEXT: s_mov_b32 s9, s3
@@ -2474,24 +2518,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB4_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s9, s5, s8
-; GFX1164-NEXT: s_mul_hi_u32 s10, s4, s8
+; GFX1164-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1164-NEXT: s_mul_i32 s10, s5, s8
; GFX1164-NEXT: s_mul_i32 s8, s4, s8
-; GFX1164-NEXT: s_add_i32 s10, s10, s9
+; GFX1164-NEXT: s_add_u32 s9, s9, s10
; GFX1164-NEXT: v_mov_b32_e32 v0, s8
-; GFX1164-NEXT: v_mov_b32_e32 v1, s10
+; GFX1164-NEXT: v_mov_b32_e32 v1, s9
+; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_mov_b32 s10, -1
; GFX1164-NEXT: s_mov_b32 s8, s2
; GFX1164-NEXT: s_mov_b32 s9, s3
@@ -2518,23 +2563,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s7, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s7
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s8, s5, s7
; GFX1132-NEXT: s_mul_hi_u32 s9, s4, s7
-; GFX1132-NEXT: s_mul_i32 s7, s4, s7
-; GFX1132-NEXT: s_add_i32 s9, s9, s8
+; GFX1132-NEXT: s_mul_i32 s10, s5, s7
+; GFX1132-NEXT: s_mul_i32 s8, s4, s7
+; GFX1132-NEXT: s_add_u32 s9, s9, s10
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s9
+; GFX1132-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_mov_b32 s10, -1
; GFX1132-NEXT: s_mov_b32 s8, s2
; GFX1132-NEXT: s_mov_b32 s9, s3
@@ -2561,23 +2606,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB4_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1264-NEXT: s_mul_i32 s10, s5, s8
+; GFX1264-NEXT: s_mul_i32 s8, s4, s8
+; GFX1264-NEXT: s_add_co_u32 s9, s9, s10
; GFX1264-NEXT: v_mov_b32_e32 v0, s8
; GFX1264-NEXT: v_mov_b32_e32 v1, s9
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -2602,30 +2649,31 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s6, exec_lo
-; GFX1232-NEXT: s_mov_b32 s7, 0
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1232-NEXT: s_mov_b32 s8, exec_lo
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB4_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s15, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1232-NEXT: s_mov_b32 s14, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s12, s2
-; GFX1232-NEXT: s_mov_b32 s13, s3
-; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1232-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1232-NEXT: s_mul_i32 s10, s5, s7
+; GFX1232-NEXT: s_mul_i32 s8, s4, s7
+; GFX1232-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
; GFX1232-NEXT: global_inv scope:SCOPE_DEV
; GFX1232-NEXT: .LBB4_2:
-; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: v_readfirstlane_b32 s3, v1
; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
@@ -2642,22 +2690,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-NEXT: s_mov_b32 s11, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB4_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1364-NEXT: s_mul_i32 s10, s5, s8
+; GFX1364-NEXT: s_mul_i32 s8, s4, s8
+; GFX1364-NEXT: s_add_co_u32 s9, s9, s10
; GFX1364-NEXT: v_mov_b32_e32 v0, s8
; GFX1364-NEXT: v_mov_b32_e32 v1, s9
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -2685,31 +2736,34 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s7, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s8, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1332-NEXT: s_mov_b32 s6, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB4_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s15, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1332-NEXT: s_mov_b32 s14, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1332-NEXT: s_mov_b32 s12, s2
-; GFX1332-NEXT: s_mov_b32 s13, s3
+; GFX1332-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1332-NEXT: s_mul_i32 s10, s5, s7
+; GFX1332-NEXT: s_mul_i32 s8, s4, s7
+; GFX1332-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s8, s2
+; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_storecnt 0x0
-; GFX1332-NEXT: buffer_atomic_add_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1332-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: .LBB4_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_readfirstlane_b32 s3, v1
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
@@ -4183,23 +4237,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-LABEL: sub_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s12, s4, 5
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s12, s12, 5
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4233,23 +4287,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX8-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s12, s4, 5
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s12, s12, 5
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4281,23 +4335,23 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s12, s4, 5
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s12, s12, 5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
@@ -4329,25 +4383,25 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: sub_i32_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB6_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1064-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s12, s4, 5
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s12, s12, 5
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_mov_b32 s5, s3
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v4, v0
@@ -4378,24 +4432,24 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i32_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1032-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1032-NEXT: s_mul_i32 s10, s4, 5
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s10, 5
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_mov_b32 s5, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v4, v0
@@ -4426,26 +4480,26 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: sub_i32_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB6_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1164-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s12, s4, 5
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s12, s12, 5
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_mov_b32 s5, s3
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4480,25 +4534,24 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i32_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB6_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1132-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1132-NEXT: s_mul_i32 s10, s4, 5
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s10, 5
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_mov_b32 s5, s3
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s4
; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4532,22 +4585,21 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: sub_i32_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB6_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s10, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: v_mov_b32_e32 v1, s6
+; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -4569,19 +4621,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i32_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB6_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_mul_i32 s5, s5, 5
-; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -4603,20 +4655,21 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: sub_i32_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB6_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: v_mov_b32_e32 v1, s6
+; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -4641,19 +4694,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: sub_i32_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB6_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_mul_i32 s5, s5, 5
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -4685,23 +4738,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s13, s12, s4
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s13, s12, s5
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4735,23 +4788,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB7_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX8-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s13, s12, s4
; GFX8-NEXT: s_mov_b64 s[10:11], 0
; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s13, s12, s5
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4784,23 +4837,23 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX9-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s13, s12, s4
; GFX9-NEXT: s_mov_b64 s[10:11], 0
; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s13, s12, s5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s5
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4834,25 +4887,25 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1064-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s13, s12, s4
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s13, s12, s5
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_mov_b32 s5, s3
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s5
+; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v4, v0
@@ -4885,24 +4938,24 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1032-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX1032-NEXT: s_mul_i32 s11, s8, s4
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s11, s8, s5
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_mov_b32 s5, s3
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v4, v0
@@ -4935,26 +4988,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1164-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s13, s12, s4
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s13, s12, s5
; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_mov_b32 s5, s3
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s5
+; GFX1164-NEXT: s_mov_b32 s5, s3
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4991,25 +5044,24 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s10, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1132-NEXT: s_mul_i32 s11, s8, s4
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s11, s8, s5
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_mov_b32 s5, s3
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s4
; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132-NEXT: s_mov_b32 s5, s3
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5044,33 +5096,32 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1264-NEXT: ; implicit-def: $vgpr1
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1264-NEXT: s_cbranch_execz .LBB7_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: s_mov_b32 s15, 0x31016000
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: s_mul_i32 s6, s8, s6
-; GFX1264-NEXT: s_mov_b32 s14, -1
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: s_mov_b32 s12, s2
-; GFX1264-NEXT: s_mov_b32 s13, s3
-; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1264-NEXT: s_mul_i32 s7, s6, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s10, -1
+; GFX1264-NEXT: s_mov_b32 s8, s2
+; GFX1264-NEXT: s_mov_b32 s9, s3
+; GFX1264-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1264-NEXT: s_wait_loadcnt 0x0
; GFX1264-NEXT: global_inv scope:SCOPE_DEV
; GFX1264-NEXT: .LBB7_2:
; GFX1264-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX1264-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX1264-NEXT: v_readfirstlane_b32 s2, v1
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -5083,22 +5134,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b32 s4, s[4:5], 0x34
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s5, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1232-NEXT: s_mov_b32 s5, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1232-NEXT: s_cbranch_execz .LBB7_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-NEXT: s_mul_i32 s6, s4, s6
-; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
; GFX1232-NEXT: v_mov_b32_e32 v1, s6
+; GFX1232-NEXT: s_mov_b32 s10, -1
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5119,34 +5168,35 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB7_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: s_mov_b32 s15, 0x31016000
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s6, s8, s6
-; GFX1364-NEXT: s_mov_b32 s14, -1
-; GFX1364-NEXT: v_mov_b32_e32 v1, s6
-; GFX1364-NEXT: s_mov_b32 s12, s2
-; GFX1364-NEXT: s_mov_b32 s13, s3
+; GFX1364-NEXT: s_mul_i32 s7, s6, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s10, -1
+; GFX1364-NEXT: s_mov_b32 s8, s2
+; GFX1364-NEXT: s_mov_b32 s9, s3
; GFX1364-NEXT: global_wb scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_storecnt 0x0
-; GFX1364-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1364-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_DEV
; GFX1364-NEXT: s_wait_loadcnt 0x0
; GFX1364-NEXT: .LBB7_2:
; GFX1364-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX1364-NEXT: v_mul_lo_u32 v0, s6, v0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -5159,20 +5209,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b32 s4, s[4:5], 0x34 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s5, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s5
+; GFX1332-NEXT: s_mov_b32 s5, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB7_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s6, s4, s6
-; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
; GFX1332-NEXT: v_mov_b32_e32 v1, s6
+; GFX1332-NEXT: s_mov_b32 s10, -1
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
@@ -6446,42 +6496,47 @@ entry:
define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace(1) %inout) {
; GFX7LESS-LABEL: sub_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX7LESS-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s5, s6, 0
+; GFX7LESS-NEXT: s_add_u32 s4, s4, s5
+; GFX7LESS-NEXT: s_mul_i32 s12, s6, 5
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
-; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mul_i32 s12, s12, 5
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v1
-; GFX7LESS-NEXT: v_subrev_i32_e32 v5, vcc, s12, v7
-; GFX7LESS-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
+; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s12, v8
+; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, v5
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v6
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v7
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
@@ -6505,41 +6560,46 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i64_constant:
; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s6
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v1
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX8-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s5, s6, 0
+; GFX8-NEXT: s_add_u32 s4, s4, s5
+; GFX8-NEXT: s_mul_i32 s12, s6, 5
; GFX8-NEXT: s_mov_b64 s[10:11], 0
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mul_i32 s12, s12, 5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
+; GFX8-NEXT: v_mov_b32_e32 v1, s15
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v7, v0
-; GFX8-NEXT: v_mov_b32_e32 v8, v1
-; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s12, v7
-; GFX8-NEXT: v_subbrev_u32_e32 v6, vcc, 0, v8, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, v5
-; GFX8-NEXT: v_mov_b32_e32 v1, v6
-; GFX8-NEXT: v_mov_b32_e32 v2, v7
-; GFX8-NEXT: v_mov_b32_e32 v3, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v8
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v1, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, v8
+; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_cbranch_execnz .LBB9_2
@@ -6563,40 +6623,44 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-LABEL: sub_i64_constant:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s12, s4, 5
+; GFX9-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX9-NEXT: s_mul_i32 s4, s4, 0
+; GFX9-NEXT: s_add_u32 s4, s5, s4
; GFX9-NEXT: s_mov_b64 s[10:11], 0
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mul_i32 s12, s12, 5
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s4
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v7, v0
-; GFX9-NEXT: v_mov_b32_e32 v8, v1
-; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s12, v7
-; GFX9-NEXT: v_subbrev_co_u32_e32 v6, vcc, 0, v8, vcc
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
-; GFX9-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-NEXT: v_mov_b32_e32 v3, v8
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s12, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-NEXT: v_mov_b32_e32 v3, v9
; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
@@ -6620,32 +6684,35 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-LABEL: sub_i64_constant:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s12, s4, 5
+; GFX1064-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1064-NEXT: s_mul_i32 s4, s4, 0
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
-; GFX1064-NEXT: s_mul_i32 s12, s12, 5
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
-; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_add_u32 s13, s5, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v7, v0
; GFX1064-NEXT: v_mov_b32_e32 v8, v1
; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s12
-; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, 0, v8, vcc
+; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s13, v8, vcc
; GFX1064-NEXT: v_mov_b32_e32 v2, v7
; GFX1064-NEXT: v_mov_b32_e32 v0, v5
; GFX1064-NEXT: v_mov_b32_e32 v3, v8
@@ -6677,31 +6744,34 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s6
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s10, 5
-; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1032-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1032-NEXT: s_mul_i32 s11, s4, 0
+; GFX1032-NEXT: s_mul_i32 s10, s4, 5
+; GFX1032-NEXT: s_add_u32 s11, s5, s11
; GFX1032-NEXT: s_mov_b32 s4, s2
; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v7, v0
; GFX1032-NEXT: v_mov_b32_e32 v8, v1
; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
-; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s11, v8, vcc_lo
; GFX1032-NEXT: v_mov_b32_e32 v2, v7
; GFX1032-NEXT: v_mov_b32_e32 v0, v5
; GFX1032-NEXT: v_mov_b32_e32 v3, v8
@@ -6733,27 +6803,30 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-LABEL: sub_i64_constant:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-NEXT: s_cbranch_execz .LBB9_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1164-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s12, s4, 5
+; GFX1164-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1164-NEXT: s_mul_i32 s4, s4, 0
; GFX1164-NEXT: s_mov_b64 s[10:11], 0
-; GFX1164-NEXT: s_mul_i32 s12, s12, 5
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
-; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_add_u32 s13, s5, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -6761,7 +6834,7 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164-NEXT: v_mov_b32_e32 v8, v1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s12
-; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc
+; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s13, v8, vcc
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1164-NEXT: v_mov_b32_e32 v0, v5
; GFX1164-NEXT: v_mov_b32_e32 v2, v7
@@ -6798,32 +6871,34 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB9_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s6
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s10, 5
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1132-NEXT: s_mul_hi_u32 s5, 5, s4
+; GFX1132-NEXT: s_mul_i32 s11, s4, 0
+; GFX1132-NEXT: s_mul_i32 s10, s4, 5
+; GFX1132-NEXT: s_add_u32 s11, s5, s11
; GFX1132-NEXT: s_mov_b32 s4, s2
; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s11, v8, vcc_lo
; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX1132-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
@@ -6856,22 +6931,24 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1264-LABEL: sub_i64_constant:
; GFX1264: ; %bb.0: ; %entry
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[4:5], exec
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1264-NEXT: s_mov_b64 s[4:5], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB9_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1264-NEXT: v_mov_b32_e32 v1, 0
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1264-NEXT: s_mul_i32 s8, s6, 0
; GFX1264-NEXT: s_mul_i32 s6, s6, 5
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_add_co_u32 s7, s7, s8
; GFX1264-NEXT: v_mov_b32_e32 v0, s6
+; GFX1264-NEXT: v_mov_b32_e32 v1, s7
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
; GFX1264-NEXT: s_mov_b32 s8, s2
@@ -6897,19 +6974,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1232-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB9_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1232-NEXT: s_mul_i32 s8, s5, 0
+; GFX1232-NEXT: s_mul_i32 s6, s5, 5
+; GFX1232-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
; GFX1232-NEXT: s_mov_b32 s9, s3
@@ -6934,20 +7015,24 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1364-LABEL: sub_i64_constant:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB9_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s7, 5, s6
+; GFX1364-NEXT: s_mul_i32 s8, s6, 0
; GFX1364-NEXT: s_mul_i32 s6, s6, 5
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
+; GFX1364-NEXT: v_mov_b32_e32 v1, s7
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_wait_kmcnt 0x0
; GFX1364-NEXT: s_mov_b32 s8, s2
@@ -6976,19 +7061,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1332-LABEL: sub_i64_constant:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s4, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB9_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1332-NEXT: s_mul_hi_u32 s7, 5, s5
+; GFX1332-NEXT: s_mul_i32 s8, s5, 0
+; GFX1332-NEXT: s_mul_i32 s6, s5, 5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1332-NEXT: s_mul_i32 s5, s5, 5
; GFX1332-NEXT: s_mov_b32 s10, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mov_b32 s8, s2
; GFX1332-NEXT: s_mov_b32 s9, s3
@@ -7021,48 +7110,49 @@ entry:
define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %subitive) {
; GFX7LESS-LABEL: sub_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s9, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
-; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s7, v0
+; GFX7LESS-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0
+; GFX7LESS-NEXT: s_mul_i32 s5, s9, s6
+; GFX7LESS-NEXT: s_add_u32 s4, s4, s5
+; GFX7LESS-NEXT: s_mul_i32 s14, s8, s6
; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s16
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s17
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v10, v1
-; GFX7LESS-NEXT: v_sub_i32_e32 v7, vcc, v9, v6
-; GFX7LESS-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
+; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s14, v8
+; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, v7
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v8
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v9
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v10
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
@@ -7071,7 +7161,7 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s9, v4
; GFX7LESS-NEXT: v_mul_hi_u32 v2, s8, v4
; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
@@ -7087,45 +7177,48 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i64_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v6, s7, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX8-NEXT: s_cbranch_execz .LBB10_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s8, v0, 0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX8-NEXT: s_mul_i32 s6, s9, s6
+; GFX8-NEXT: s_load_dwordx2 s[16:17], s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s5, s9, s6
+; GFX8-NEXT: s_add_u32 s4, s4, s5
+; GFX8-NEXT: s_mul_i32 s14, s8, s6
; GFX8-NEXT: s_mov_b64 s[12:13], 0
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v5
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s16
+; GFX8-NEXT: v_mov_b32_e32 v1, s17
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_mov_b32 s4, s2
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v9, v0
-; GFX8-NEXT: v_mov_b32_e32 v10, v1
-; GFX8-NEXT: v_sub_u32_e32 v7, vcc, v9, v4
-; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, v7
-; GFX8-NEXT: v_mov_b32_e32 v1, v8
-; GFX8-NEXT: v_mov_b32_e32 v2, v9
-; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s14, v8
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v1, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, v8
+; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8-NEXT: s_cbranch_execnz .LBB10_2
@@ -7133,13 +7226,12 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
; GFX8-NEXT: .LBB10_4: ; %Flow4
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v4, s9, v6
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v6, 0
+; GFX8-NEXT: v_mul_lo_u32 v5, s9, v4
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v4, 0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4
+; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v5
; GFX8-NEXT: v_mov_b32_e32 v3, s4
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v2
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -7151,26 +7243,26 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX9-NEXT: s_mul_i32 s7, s9, s6
-; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
-; GFX9-NEXT: s_add_i32 s7, s12, s7
-; GFX9-NEXT: s_mul_i32 s14, s8, s6
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX9-NEXT: s_mul_i32 s14, s8, s4
+; GFX9-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX9-NEXT: s_mul_i32 s4, s9, s4
+; GFX9-NEXT: s_add_u32 s4, s5, s4
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s4
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
@@ -7214,29 +7306,29 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1064-NEXT: s_mul_i32 s14, s8, s4
+; GFX1064-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1064-NEXT: s_mul_i32 s4, s9, s4
; GFX1064-NEXT: s_mov_b64 s[12:13], 0
-; GFX1064-NEXT: s_mul_i32 s7, s9, s6
-; GFX1064-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1064-NEXT: s_mul_i32 s14, s8, s6
-; GFX1064-NEXT: s_add_i32 s15, s15, s7
-; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1064-NEXT: s_mov_b32 s6, -1
-; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s4
-; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_add_u32 s15, s5, s4
; GFX1064-NEXT: s_mov_b32 s4, s2
; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s6
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: v_mov_b32_e32 v7, v0
@@ -7276,28 +7368,28 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s11, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s10, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1032-NEXT: s_mul_i32 s7, s9, s6
-; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1032-NEXT: s_mul_i32 s12, s8, s6
-; GFX1032-NEXT: s_add_i32 s13, s13, s7
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s4
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
+; GFX1032-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1032-NEXT: s_mul_i32 s13, s9, s4
+; GFX1032-NEXT: s_mul_i32 s12, s8, s4
+; GFX1032-NEXT: s_add_u32 s13, s5, s13
; GFX1032-NEXT: s_mov_b32 s4, s2
; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
+; GFX1032-NEXT: v_mov_b32_e32 v1, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: v_mov_b32_e32 v7, v0
@@ -7337,30 +7429,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-NEXT: s_mov_b64 s[10:11], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-NEXT: s_cbranch_execz .LBB10_4
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1164-NEXT: s_mul_i32 s14, s8, s4
+; GFX1164-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1164-NEXT: s_mul_i32 s4, s9, s4
; GFX1164-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-NEXT: s_mul_i32 s7, s9, s6
-; GFX1164-NEXT: s_mul_hi_u32 s15, s8, s6
-; GFX1164-NEXT: s_mul_i32 s14, s8, s6
-; GFX1164-NEXT: s_add_i32 s15, s15, s7
-; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1164-NEXT: s_mov_b32 s6, -1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v0, s4
-; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_add_u32 s15, s5, s4
; GFX1164-NEXT: s_mov_b32 s4, s2
; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s6
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -7407,29 +7499,27 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: s_mov_b32 s11, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132-NEXT: s_mov_b32 s10, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB10_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_mul_i32 s7, s9, s6
-; GFX1132-NEXT: s_mul_hi_u32 s13, s8, s6
-; GFX1132-NEXT: s_mul_i32 s12, s8, s6
-; GFX1132-NEXT: s_add_i32 s13, s13, s7
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_load_b64 s[6:7], s[2:3], 0x0
+; GFX1132-NEXT: s_mul_hi_u32 s5, s8, s4
+; GFX1132-NEXT: s_mul_i32 s13, s9, s4
+; GFX1132-NEXT: s_mul_i32 s12, s8, s4
+; GFX1132-NEXT: s_add_u32 s13, s5, s13
; GFX1132-NEXT: s_mov_b32 s4, s2
; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7471,23 +7561,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1264-NEXT: s_clause 0x1
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1264-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-NEXT: s_mov_b32 s11, 0
-; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1264-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-NEXT: s_mov_b64 s[6:7], exec
-; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1264-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1264-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1264-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1264-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1264-NEXT: s_cbranch_execz .LBB10_2
; GFX1264-NEXT: ; %bb.1:
-; GFX1264-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1264-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1264-NEXT: s_mul_i32 s10, s5, s8
+; GFX1264-NEXT: s_mul_i32 s8, s4, s8
+; GFX1264-NEXT: s_add_co_u32 s9, s9, s10
; GFX1264-NEXT: v_mov_b32_e32 v0, s8
; GFX1264-NEXT: v_mov_b32_e32 v1, s9
+; GFX1264-NEXT: s_mov_b32 s11, 0x31016000
; GFX1264-NEXT: s_mov_b32 s10, -1
; GFX1264-NEXT: s_mov_b32 s8, s2
; GFX1264-NEXT: s_mov_b32 s9, s3
@@ -7514,30 +7606,31 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: s_clause 0x1
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1232-NEXT: s_mov_b32 s6, exec_lo
-; GFX1232-NEXT: s_mov_b32 s7, 0
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1232-NEXT: s_mov_b32 s8, exec_lo
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB10_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1232-NEXT: s_mov_b32 s15, 0x31016000
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1232-NEXT: s_mov_b32 s14, -1
-; GFX1232-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1232-NEXT: s_mov_b32 s12, s2
-; GFX1232-NEXT: s_mov_b32 s13, s3
-; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1232-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1232-NEXT: s_mul_i32 s10, s5, s7
+; GFX1232-NEXT: s_mul_i32 s8, s4, s7
+; GFX1232-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1232-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: s_mov_b32 s9, s3
+; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
; GFX1232-NEXT: global_inv scope:SCOPE_DEV
; GFX1232-NEXT: .LBB10_2:
-; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
; GFX1232-NEXT: v_readfirstlane_b32 s2, v0
@@ -7556,22 +7649,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1364-NEXT: s_clause 0x1
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
-; GFX1364-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-NEXT: s_mov_b32 s11, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
+; GFX1364-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB10_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s10, s[8:9]
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[8:9], s[4:5], s[10:11]
-; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1364-NEXT: s_mul_hi_u32 s9, s4, s8
+; GFX1364-NEXT: s_mul_i32 s10, s5, s8
+; GFX1364-NEXT: s_mul_i32 s8, s4, s8
+; GFX1364-NEXT: s_add_co_u32 s9, s9, s10
; GFX1364-NEXT: v_mov_b32_e32 v0, s8
; GFX1364-NEXT: v_mov_b32_e32 v1, s9
+; GFX1364-NEXT: s_mov_b32 s11, 0x31016000
; GFX1364-NEXT: s_mov_b32 s10, -1
; GFX1364-NEXT: s_mov_b32 s8, s2
; GFX1364-NEXT: s_mov_b32 s9, s3
@@ -7601,31 +7697,34 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1332-NEXT: s_clause 0x1
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-NEXT: s_load_b64 s[4:5], s[4:5], 0x34 nv
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s7, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s8, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s7, s6
+; GFX1332-NEXT: s_mov_b32 s6, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB10_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1332-NEXT: s_mov_b32 s15, 0x31016000
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[6:7], s[4:5], s[6:7]
-; GFX1332-NEXT: s_mov_b32 s14, -1
-; GFX1332-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX1332-NEXT: s_mov_b32 s12, s2
-; GFX1332-NEXT: s_mov_b32 s13, s3
+; GFX1332-NEXT: s_mul_hi_u32 s9, s4, s7
+; GFX1332-NEXT: s_mul_i32 s10, s5, s7
+; GFX1332-NEXT: s_mul_i32 s8, s4, s7
+; GFX1332-NEXT: s_add_co_u32 s9, s9, s10
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1332-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1332-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1332-NEXT: s_mov_b32 s10, -1
+; GFX1332-NEXT: s_mov_b32 s8, s2
+; GFX1332-NEXT: s_mov_b32 s9, s3
; GFX1332-NEXT: global_wb scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_storecnt 0x0
-; GFX1332-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[12:15], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1332-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_DEV
; GFX1332-NEXT: s_wait_loadcnt 0x0
; GFX1332-NEXT: .LBB10_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_mad_co_u64_u32 v[3:4], null, s4, v2, 0
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
@@ -9386,15 +9485,16 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_and_b32 s5, s10, 3
; GFX7LESS-NEXT: s_and_b32 s0, s14, 0xff
+; GFX7LESS-NEXT: s_lshl_b32 s10, s5, 3
; GFX7LESS-NEXT: s_lshl_b32 s15, s0, s10
; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
@@ -9438,15 +9538,16 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX8-NEXT: s_and_b32 s4, s10, -4
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s10, s0, 3
+; GFX8-NEXT: s_and_b32 s5, s10, 3
; GFX8-NEXT: s_and_b32 s0, s14, 0xff
+; GFX8-NEXT: s_lshl_b32 s10, s5, 3
; GFX8-NEXT: s_lshl_b32 s15, s0, s10
; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
@@ -9489,15 +9590,16 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX9-NEXT: s_and_b32 s4, s10, -4
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s10, s0, 3
+; GFX9-NEXT: s_and_b32 s5, s10, 3
; GFX9-NEXT: s_and_b32 s0, s14, 0xff
+; GFX9-NEXT: s_lshl_b32 s10, s5, 3
; GFX9-NEXT: s_lshl_b32 s15, s0, s10
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
@@ -9542,12 +9644,13 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_and_b32 s5, s14, 0xff
; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, s14, 0xff
; GFX1064-NEXT: s_mov_b64 s[12:13], 0
-; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1064-NEXT: s_lshl_b32 s15, s5, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9591,14 +9694,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_and_b32 s6, s1, 0xff
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX1032-NEXT: s_and_b32 s5, s10, 3
-; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1032-NEXT: s_and_b32 s5, s1, 0xff
-; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
; GFX1032-NEXT: s_mov_b32 s5, s11
+; GFX1032-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9645,12 +9748,13 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_and_b32 s5, s14, 0xff
; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9700,12 +9804,13 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_and_b32 s5, s14, 0xff
; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9753,15 +9858,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9808,15 +9912,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9865,12 +9968,13 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_and_b32 s5, s14, 0xff
; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, s14, 0xff
; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1264-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9922,12 +10026,13 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_and_b32 s5, s14, 0xff
; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, s14, 0xff
; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1264-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -9977,15 +10082,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s5, s1, 0xff
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -10034,15 +10138,14 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s5, s1, 0xff
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -10092,10 +10195,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-TRUE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1364-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
; GFX1364-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s6, s13
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -10146,10 +10249,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-FAKE16-NEXT: s_and_b32 s6, s12, 0xff
; GFX1364-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
; GFX1364-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s6, s13
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1364-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -10200,11 +10303,11 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1332-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s7, s10
+; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1332-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -10252,11 +10355,11 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-FAKE16-NEXT: s_and_b32 s7, s1, 0xff
; GFX1332-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
-; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s7, s10
+; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1332-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
@@ -10296,17 +10399,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s12, s4
+; GFX7LESS-NEXT: s_sext_i32_i8 s5, s12
+; GFX7LESS-NEXT: s_mul_i32 s6, s5, s4
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
@@ -10357,17 +10461,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB13_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s12, s4
+; GFX8-NEXT: s_sext_i32_i8 s5, s12
+; GFX8-NEXT: s_mul_i32 s6, s5, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
@@ -10416,17 +10521,18 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_4
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s12, s4
+; GFX9-NEXT: s_sext_i32_i8 s5, s12
+; GFX9-NEXT: s_mul_i32 s6, s5, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
@@ -10475,9 +10581,10 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
@@ -10488,11 +10595,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_sext_i32_i8 s7, s12
; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_mul_i32 s7, s7, s6
; GFX1064-NEXT: s_lshl_b32 s13, 0xff, s2
-; GFX1064-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1064-NEXT: s_and_b32 s6, s7, 0xff
; GFX1064-NEXT: s_not_b32 s14, s13
; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
@@ -10533,9 +10640,10 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
@@ -10546,11 +10654,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_sext_i32_i8 s7, s8
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_mul_i32 s7, s7, s6
; GFX1032-NEXT: s_lshl_b32 s11, 0xff, s2
-; GFX1032-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1032-NEXT: s_and_b32 s6, s7, 0xff
; GFX1032-NEXT: s_not_b32 s12, s11
; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
@@ -10590,11 +10698,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10604,11 +10713,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-TRUE16-NEXT: s_sext_i32_i8 s7, s12
; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
-; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
@@ -10654,11 +10763,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10668,11 +10778,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-FAKE16-NEXT: s_sext_i32_i8 s7, s12
; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
-; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
@@ -10718,12 +10828,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
@@ -10732,11 +10842,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-TRUE16-NEXT: s_sext_i32_i8 s7, s8
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10779,12 +10889,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
@@ -10793,11 +10903,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-FAKE16-NEXT: s_sext_i32_i8 s7, s8
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10840,11 +10950,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10854,15 +10965,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-TRUE16-NEXT: s_sext_i32_i8 s7, s12
; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xff, s2
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10907,11 +11015,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -10921,15 +11030,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-FAKE16-NEXT: s_sext_i32_i8 s7, s12
; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xff, s2
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -10974,12 +11080,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
@@ -10988,15 +11094,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-TRUE16-NEXT: s_sext_i32_i8 s7, s8
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xff, s2
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -11039,12 +11142,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
@@ -11053,15 +11156,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-FAKE16-NEXT: s_sext_i32_i8 s7, s8
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xff, s2
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s7, 0xff
; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -11104,11 +11204,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1364-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-TRUE16-NEXT: s_cbranch_execz .LBB13_4
@@ -11117,11 +11218,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-TRUE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1364-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-TRUE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1364-TRUE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1364-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -11165,11 +11266,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_clause 0x1
; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1364-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-FAKE16-NEXT: s_cbranch_execz .LBB13_4
@@ -11178,11 +11280,11 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-FAKE16-NEXT: s_sext_i32_i8 s7, s10
; GFX1364-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-FAKE16-NEXT: s_lshl_b32 s12, 0xff, s11
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX1364-FAKE16-NEXT: s_and_b32 s2, s7, 0xff
; GFX1364-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -11226,12 +11328,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-TRUE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1332-TRUE16-NEXT: ; %bb.1:
@@ -11239,9 +11341,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-TRUE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1332-TRUE16-NEXT: s_not_b32 s11, s3
@@ -11285,12 +11387,12 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_clause 0x1
; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-FAKE16-NEXT: s_cbranch_execz .LBB13_4
; GFX1332-FAKE16-NEXT: ; %bb.1:
@@ -11298,9 +11400,9 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-FAKE16-NEXT: s_sext_i32_i8 s11, s8
; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xff, s2
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s6, 0xff
; GFX1332-FAKE16-NEXT: s_not_b32 s11, s3
@@ -11821,15 +11923,16 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
-; GFX7LESS-NEXT: s_lshl_b32 s10, s0, 3
+; GFX7LESS-NEXT: s_and_b32 s5, s10, 3
; GFX7LESS-NEXT: s_and_b32 s0, s14, 0xffff
+; GFX7LESS-NEXT: s_lshl_b32 s10, s5, 3
; GFX7LESS-NEXT: s_lshl_b32 s15, s0, s10
; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
; GFX7LESS-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_or_b32_e32 v0, s15, v1
@@ -11873,15 +11976,16 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX8-NEXT: s_and_b32 s4, s10, -4
; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT: s_and_b32 s0, s10, 3
-; GFX8-NEXT: s_lshl_b32 s10, s0, 3
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX8-NEXT: s_and_b32 s5, s10, 3
+; GFX8-NEXT: s_and_b32 s0, s14, 0xffff
+; GFX8-NEXT: s_lshl_b32 s10, s5, 3
; GFX8-NEXT: s_lshl_b32 s15, s0, s10
; GFX8-NEXT: s_mov_b64 s[12:13], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s11
; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_or_b32_e32 v0, s15, v1
@@ -11924,15 +12028,16 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX9-NEXT: s_and_b32 s4, s10, -4
; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
-; GFX9-NEXT: s_and_b32 s0, s10, 3
-; GFX9-NEXT: s_lshl_b32 s10, s0, 3
-; GFX9-NEXT: s_and_b32 s0, 0xffff, s14
+; GFX9-NEXT: s_and_b32 s5, s10, 3
+; GFX9-NEXT: s_and_b32 s0, s14, 0xffff
+; GFX9-NEXT: s_lshl_b32 s10, s5, 3
; GFX9-NEXT: s_lshl_b32 s15, s0, s10
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s11
; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_or_b32_e32 v0, s15, v1
@@ -11977,12 +12082,13 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_and_b32 s1, s10, 3
; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_and_b32 s5, s14, 0xffff
; GFX1064-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1064-NEXT: s_and_b32 s1, 0xffff, s14
; GFX1064-NEXT: s_mov_b64 s[12:13], 0
-; GFX1064-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1064-NEXT: s_lshl_b32 s15, s5, s10
; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_mov_b32 s5, s11
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s0
; GFX1064-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12026,14 +12132,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_and_b32 s4, s10, -4
; GFX1032-NEXT: s_mov_b32 s5, s11
-; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_and_b32 s6, s1, 0xffff
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX1032-NEXT: s_and_b32 s5, s10, 3
-; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1032-NEXT: s_and_b32 s5, 0xffff, s1
-; GFX1032-NEXT: s_lshl_b32 s12, s5, s10
; GFX1032-NEXT: s_mov_b32 s5, s11
+; GFX1032-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1032-NEXT: s_mov_b32 s6, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s0
; GFX1032-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12080,12 +12186,13 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_and_b32 s5, s14, 0xffff
; GFX1164-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
; GFX1164-TRUE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12135,12 +12242,13 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_and_b32 s5, s14, 0xffff
; GFX1164-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
; GFX1164-FAKE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12188,15 +12296,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xffff
; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1132-TRUE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12243,15 +12350,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xffff
; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1132-FAKE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1132-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12300,12 +12406,13 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_and_b32 s5, s14, 0xffff
; GFX1264-TRUE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s14
; GFX1264-TRUE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1264-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12357,12 +12464,13 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_and_b32 s5, s14, 0xffff
; GFX1264-FAKE16-NEXT: s_lshl_b32 s10, s1, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s14
; GFX1264-FAKE16-NEXT: s_mov_b64 s[12:13], 0
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s1, s10
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s5, s10
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1264-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12412,15 +12520,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xffff
; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1232-TRUE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s5, 0xffff, s1
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12469,15 +12576,14 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xffff
; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX1232-FAKE16-NEXT: s_and_b32 s5, s10, 3
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s5, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s5, 0xffff, s1
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s5, s10
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s12, s6, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1232-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12527,10 +12633,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-TRUE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1364-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
; GFX1364-TRUE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s6, s13
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1364-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1364-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -12581,10 +12687,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s1, s10, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1364-FAKE16-NEXT: s_and_b32 s6, s12, 0xffff
; GFX1364-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
-; GFX1364-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
; GFX1364-FAKE16-NEXT: s_mov_b64 s[10:11], 0
-; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s6, s13
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1364-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1364-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -12635,11 +12741,11 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-TRUE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1332-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
-; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-TRUE16-NEXT: s_lshl_b32 s11, s7, s10
+; GFX1332-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1332-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12687,11 +12793,11 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[10:11], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s6, s10, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1332-FAKE16-NEXT: s_and_b32 s7, s1, 0xffff
; GFX1332-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
-; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1332-FAKE16-NEXT: s_lshl_b32 s11, s7, s10
+; GFX1332-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1332-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1332-FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1332-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
@@ -12731,24 +12837,25 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s12, s4
+; GFX7LESS-NEXT: s_sext_i32_i16 s5, s12
+; GFX7LESS-NEXT: s_mul_i32 s5, s5, s4
+; GFX7LESS-NEXT: s_and_b32 s6, s5, 0xffff
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX7LESS-NEXT: s_and_b32 s6, s6, 0xffff
; GFX7LESS-NEXT: s_not_b32 s14, s13
; GFX7LESS-NEXT: s_lshl_b32 s15, s6, s2
; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
@@ -12792,24 +12899,25 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX8-NEXT: s_cbranch_execz .LBB16_4
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s6, s12, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s12
+; GFX8-NEXT: s_mul_i32 s6, s5, s4
; GFX8-NEXT: s_and_b32 s4, s2, -4
; GFX8-NEXT: s_mov_b32 s5, s3
; GFX8-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX8-NEXT: s_and_b32 s2, s2, 3
; GFX8-NEXT: s_lshl_b32 s2, s2, 3
; GFX8-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
; GFX8-NEXT: s_not_b32 s14, s13
; GFX8-NEXT: s_lshl_b32 s15, s6, s2
; GFX8-NEXT: s_mov_b64 s[10:11], 0
@@ -12851,24 +12959,25 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_4
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s6, s12, s4
+; GFX9-NEXT: s_sext_i32_i16 s5, s12
+; GFX9-NEXT: s_mul_i32 s6, s5, s4
; GFX9-NEXT: s_and_b32 s4, s2, -4
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: s_load_dword s5, s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s2, s2, 3
; GFX9-NEXT: s_lshl_b32 s2, s2, 3
; GFX9-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX9-NEXT: s_and_b32 s6, 0xffff, s6
; GFX9-NEXT: s_not_b32 s14, s13
; GFX9-NEXT: s_lshl_b32 s15, s6, s2
; GFX9-NEXT: s_mov_b64 s[10:11], 0
@@ -12910,9 +13019,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
@@ -12923,11 +13033,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1064-NEXT: s_mov_b32 s5, s3
; GFX1064-NEXT: s_and_b32 s2, s2, 3
; GFX1064-NEXT: s_load_dword s5, s[4:5], 0x0
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_sext_i32_i16 s7, s12
; GFX1064-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1064-NEXT: s_mul_i32 s6, s12, s6
+; GFX1064-NEXT: s_mul_i32 s7, s7, s6
; GFX1064-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX1064-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1064-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1064-NEXT: s_not_b32 s14, s13
; GFX1064-NEXT: s_lshl_b32 s15, s6, s2
; GFX1064-NEXT: s_mov_b64 s[10:11], 0
@@ -12968,9 +13078,10 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: s_mov_b32 s10, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
@@ -12981,11 +13092,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: s_and_b32 s2, s2, 3
; GFX1032-NEXT: s_load_dword s5, s[4:5], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_sext_i32_i16 s7, s8
; GFX1032-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1032-NEXT: s_mul_i32 s6, s8, s6
+; GFX1032-NEXT: s_mul_i32 s7, s7, s6
; GFX1032-NEXT: s_lshl_b32 s11, 0xffff, s2
-; GFX1032-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1032-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1032-NEXT: s_not_b32 s12, s11
; GFX1032-NEXT: s_lshl_b32 s13, s6, s2
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
@@ -13025,11 +13136,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_clause 0x1
; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1164-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13039,11 +13151,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1164-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1164-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-TRUE16-NEXT: s_sext_i32_i16 s7, s12
; GFX1164-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1164-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX1164-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1164-TRUE16-NEXT: s_not_b32 s14, s13
; GFX1164-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
@@ -13089,11 +13201,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_clause 0x1
; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1164-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1164-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1164-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13103,11 +13216,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1164-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1164-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1164-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-FAKE16-NEXT: s_sext_i32_i16 s7, s12
; GFX1164-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1164-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1164-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX1164-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1164-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1164-FAKE16-NEXT: s_not_b32 s14, s13
; GFX1164-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
@@ -13153,12 +13266,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
@@ -13167,11 +13280,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1132-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1132-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-TRUE16-NEXT: s_sext_i32_i16 s7, s8
; GFX1132-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1132-TRUE16-NEXT: s_not_b32 s12, s11
; GFX1132-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13214,12 +13327,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1132-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
@@ -13228,11 +13341,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1132-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1132-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1132-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-FAKE16-NEXT: s_sext_i32_i16 s7, s8
; GFX1132-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1132-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1132-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1132-FAKE16-NEXT: s_not_b32 s12, s11
; GFX1132-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13275,11 +13388,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_clause 0x1
; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1264-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1264-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13289,15 +13403,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1264-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1264-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-TRUE16-NEXT: s_sext_i32_i16 s7, s12
; GFX1264-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1264-TRUE16-NEXT: s_not_b32 s14, s13
-; GFX1264-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-TRUE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13342,11 +13453,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_clause 0x1
; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
-; GFX1264-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1264-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1264-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13356,15 +13468,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1264-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1264-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1264-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1264-FAKE16-NEXT: s_sext_i32_i16 s7, s12
; GFX1264-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_mul_i32 s6, s12, s6
+; GFX1264-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, 0xffff, s2
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1264-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1264-FAKE16-NEXT: s_not_b32 s14, s13
-; GFX1264-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1264-FAKE16-NEXT: s_lshl_b32 s15, s6, s2
; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13409,12 +13518,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
@@ -13423,15 +13532,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
; GFX1232-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1232-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-TRUE16-NEXT: s_sext_i32_i16 s7, s8
; GFX1232-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, 0xffff, s2
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1232-TRUE16-NEXT: s_not_b32 s12, s11
-; GFX1232-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-TRUE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -13474,12 +13580,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1232-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
@@ -13488,15 +13594,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s3
; GFX1232-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1232-FAKE16-NEXT: s_load_b32 s5, s[4:5], 0x0
-; GFX1232-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1232-FAKE16-NEXT: s_sext_i32_i16 s7, s8
; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1232-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, 0xffff, s2
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s7
; GFX1232-FAKE16-NEXT: s_not_b32 s12, s11
-; GFX1232-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1232-FAKE16-NEXT: s_lshl_b32 s13, s6, s2
; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
@@ -13539,11 +13642,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-TRUE16-NEXT: s_clause 0x1
; GFX1364-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-TRUE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-TRUE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-TRUE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-TRUE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1364-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-TRUE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13552,11 +13656,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-TRUE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-TRUE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-TRUE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1364-TRUE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-TRUE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-TRUE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-TRUE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1364-TRUE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1364-TRUE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1364-TRUE16-NEXT: s_not_b32 s13, s12
; GFX1364-TRUE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13600,11 +13704,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-FAKE16-NEXT: s_clause 0x1
; GFX1364-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1364-FAKE16-NEXT: s_load_b32 s10, s[4:5], 0x34 nv
-; GFX1364-FAKE16-NEXT: s_mov_b64 s[6:7], exec
+; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1364-FAKE16-NEXT: s_mov_b64 s[4:5], exec
; GFX1364-FAKE16-NEXT: s_mov_b64 s[8:9], exec
-; GFX1364-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX1364-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX1364-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
; GFX1364-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1364-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1364-FAKE16-NEXT: s_cbranch_execz .LBB16_4
@@ -13613,11 +13718,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1364-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1364-FAKE16-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX1364-FAKE16-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1364-FAKE16-NEXT: s_sext_i32_i16 s7, s10
; GFX1364-FAKE16-NEXT: s_lshl_b32 s11, s2, 3
-; GFX1364-FAKE16-NEXT: s_mul_i32 s2, s10, s6
+; GFX1364-FAKE16-NEXT: s_mul_i32 s7, s7, s6
; GFX1364-FAKE16-NEXT: s_lshl_b32 s12, 0xffff, s11
-; GFX1364-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1364-FAKE16-NEXT: s_and_b32 s2, 0xffff, s7
; GFX1364-FAKE16-NEXT: s_not_b32 s13, s12
; GFX1364-FAKE16-NEXT: s_lshl_b32 s14, s2, s11
; GFX1364-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
@@ -13661,12 +13766,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-TRUE16-NEXT: s_clause 0x1
; GFX1332-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-TRUE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-TRUE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-TRUE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-TRUE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-TRUE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1332-TRUE16-NEXT: ; %bb.1:
@@ -13674,11 +13779,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-TRUE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-TRUE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-TRUE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-TRUE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1332-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-TRUE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-TRUE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-TRUE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1332-TRUE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1332-TRUE16-NEXT: s_not_b32 s11, s3
; GFX1332-TRUE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
@@ -13720,12 +13825,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-FAKE16-NEXT: s_clause 0x1
; GFX1332-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1332-FAKE16-NEXT: s_load_b32 s8, s[4:5], 0x34 nv
-; GFX1332-FAKE16-NEXT: s_mov_b32 s6, exec_lo
+; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1332-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-FAKE16-NEXT: s_mov_b32 s10, 0
-; GFX1332-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s4
; GFX1332-FAKE16-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1332-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-FAKE16-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1332-FAKE16-NEXT: s_cbranch_execz .LBB16_4
; GFX1332-FAKE16-NEXT: ; %bb.1:
@@ -13733,11 +13838,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1332-FAKE16-NEXT: s_and_b64 s[4:5], s[2:3], -4
; GFX1332-FAKE16-NEXT: s_and_b32 s2, s2, 3
; GFX1332-FAKE16-NEXT: s_load_b32 s7, s[4:5], 0x0
-; GFX1332-FAKE16-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1332-FAKE16-NEXT: s_sext_i32_i16 s11, s8
; GFX1332-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s8, s6
+; GFX1332-FAKE16-NEXT: s_mul_i32 s6, s11, s6
; GFX1332-FAKE16-NEXT: s_lshl_b32 s3, 0xffff, s2
-; GFX1332-FAKE16-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1332-FAKE16-NEXT: s_and_b32 s6, 0xffff, s6
; GFX1332-FAKE16-NEXT: s_not_b32 s11, s3
; GFX1332-FAKE16-NEXT: s_lshl_b32 s12, s6, s2
; GFX1332-FAKE16-NEXT: s_mov_b32 s6, -1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index c424df950c5c0..6f7b956857742 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -26,15 +26,15 @@ declare i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: add_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
@@ -54,15 +54,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -82,15 +82,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -109,17 +109,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: add_i32_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: v_mov_b32_e32 v2, s2
; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -138,16 +138,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i32_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s1
; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -166,19 +166,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: add_i32_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: v_mov_b32_e32 v2, s2
; GFX1164-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -196,17 +196,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i32_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1132-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -224,19 +224,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: add_i32_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: v_mov_b32_e32 v2, s2
; GFX1364-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
@@ -254,17 +254,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: add_i32_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s1, s1, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1332-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1332-NEXT: s_wait_dscnt 0x0
@@ -288,20 +288,20 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive) {
; GFX7LESS-LABEL: add_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_load_dword s2, s[4:5], 0xb
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
@@ -309,7 +309,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -319,20 +319,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -340,7 +340,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -350,27 +350,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -380,20 +380,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1064-LABEL: add_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -401,9 +401,10 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT: s_mov_b32 null, 0
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -412,17 +413,17 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1032-LABEL: add_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s2, s0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s2
; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -441,33 +442,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1164-LABEL: add_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB1_2:
; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v1
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0
; GFX1164-NEXT: s_endpgm
@@ -475,15 +475,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1132-LABEL: add_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s2, s0, s2
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -504,33 +504,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1364-LABEL: add_i32_uniform:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x2c nv
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB1_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s2, s6, s2
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mov_b32_e32 v2, s2
+; GFX1364-NEXT: s_mul_i32 s3, s2, s3
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: v_mov_b32_e32 v2, s3
; GFX1364-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB1_2:
; GFX1364-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1364-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX1364-NEXT: v_readfirstlane_b32 s4, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1364-NEXT: s_wait_kmcnt 0x0
+; GFX1364-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX1364-NEXT: s_mov_b32 s2, -1
; GFX1364-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1364-NEXT: s_endpgm
@@ -538,15 +537,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
; GFX1332-LABEL: add_i32_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
-; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s1, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB1_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s2, s0, s2
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1280,112 +1279,102 @@ entry:
define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX7LESS_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX7LESS_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: .LBB3_4:
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
-; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX8_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX8_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX8_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX8_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: .LBB3_4:
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
-; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX9_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX9_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX9_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX9_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB3_4:
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1064_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1064_ITERATIVE-NEXT: .LBB3_4:
@@ -1393,26 +1382,24 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1032_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s2, s1
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1032_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s1, s1, s2
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s1, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s0
-; GFX1032_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1032_ITERATIVE-NEXT: .LBB3_4:
@@ -1420,30 +1407,28 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1164_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1164_ITERATIVE-NEXT: .LBB3_4:
@@ -1451,28 +1436,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: add_i32_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1132_ITERATIVE-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1132_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB3_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1132_ITERATIVE-NEXT: ds_add_u32 v0, v1
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
+; GFX1132_ITERATIVE-NEXT: ds_add_u32 v1, v0
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1132_ITERATIVE-NEXT: .LBB3_4:
@@ -1488,31 +1471,31 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX8_DPP-LABEL: add_i32_varying_nouse:
; GFX8_DPP: ; %bb.0: ; %entry
-; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b32 s0, s2
-; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX8_DPP-NEXT: ; %bb.1:
; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX8_DPP-NEXT: s_mov_b32 m0, -1
; GFX8_DPP-NEXT: ds_add_u32 v2, v0
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1521,31 +1504,31 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX9_DPP-LABEL: add_i32_varying_nouse:
; GFX9_DPP: ; %bb.0: ; %entry
-; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b32 s0, s2
-; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX9_DPP-NEXT: ; %bb.1:
; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9_DPP-NEXT: ds_add_u32 v2, v0
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_DPP-NEXT: .LBB3_2:
@@ -1555,27 +1538,34 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1064_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_i32 s0, s2, s3
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, s0
-; GFX1064_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064_DPP-NEXT: ds_add_u32 v4, v0
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB3_2:
@@ -1585,21 +1575,24 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1032_DPP: ; %bb.0: ; %entry
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_DPP-NEXT: ds_add_u32 v3, v0
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB3_2:
@@ -1611,33 +1604,41 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164_DPP-NEXT: ds_add_u32 v4, v0
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB3_2:
@@ -1650,24 +1651,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132_DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: ds_add_u32 v0, v3
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1132_DPP-NEXT: ds_add_u32 v3, v0
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
; GFX1132_DPP-NEXT: .LBB3_2:
@@ -1679,33 +1682,40 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_permlane64_b32 v2, v1
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_mov_b32 s0, s2
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB3_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_add_u32 v0, v3
+; GFX1364-NEXT: v_mov_b32_e32 v0, s0
+; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: ds_add_u32 v4, v0
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB3_2:
@@ -1718,24 +1728,26 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: v_readlane_b32 s1, v1, 31
; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s0, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB3_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: ds_add_u32 v0, v3
+; GFX1332-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1332-NEXT: ds_add_u32 v3, v0
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB3_2:
@@ -1749,20 +1761,25 @@ entry:
define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: add_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: s_mul_i32 s6, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s2, s2, 0
+; GFX7LESS-NEXT: s_add_u32 s7, s3, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
; GFX7LESS-NEXT: s_mov_b32 m0, -1
-; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB4_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1782,20 +1799,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: add_i64_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mul_i32 s6, s2, 5
+; GFX8-NEXT: s_mul_i32 s2, s2, 0
+; GFX8-NEXT: s_add_u32 s7, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB4_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1815,19 +1837,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: add_i64_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX9-NEXT: s_mul_i32 s6, s2, 5
+; GFX9-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX9-NEXT: s_mul_i32 s2, s2, 0
+; GFX9-NEXT: s_add_u32 s7, s3, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB4_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -1846,19 +1872,23 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: add_i64_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1064-NEXT: s_mul_i32 s6, s2, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: s_add_u32 s3, s3, s6
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s2
-; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB4_2:
@@ -1876,18 +1906,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
-; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1032-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1032-NEXT: s_mul_i32 s6, s1, 0
+; GFX1032-NEXT: s_mul_i32 s2, s1, 5
+; GFX1032-NEXT: s_add_u32 s3, s3, s6
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB4_2:
@@ -1905,21 +1939,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: add_i64_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB4_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1164-NEXT: s_mul_i32 s6, s2, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_add_u32 s3, s3, s6
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB4_2:
@@ -1937,20 +1975,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
-; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1132-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1132-NEXT: s_mul_i32 s6, s1, 0
+; GFX1132-NEXT: s_mul_i32 s2, s1, 5
+; GFX1132-NEXT: s_add_u32 s3, s3, s6
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1132-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB4_2:
@@ -1968,21 +2008,25 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: add_i64_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB4_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1364-NEXT: s_mul_i32 s6, s2, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s2
-; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB4_2:
@@ -2000,20 +2044,22 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: add_i64_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB4_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: v_mov_b32_e32 v1, 0
-; GFX1332-NEXT: s_mul_i32 s1, s1, 5
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_mov_b32_e32 v0, s1
-; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1332-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1332-NEXT: s_mul_i32 s6, s1, 0
+; GFX1332-NEXT: s_mul_i32 s2, s1, 5
+; GFX1332-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1332-NEXT: v_mov_b32_e32 v1, s3
+; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB4_2:
@@ -2038,29 +2084,30 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX7LESS-LABEL: add_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s8, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s3, s6
+; GFX7LESS-NEXT: s_add_u32 s9, s7, s6
; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s9
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB5_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s4, s0
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v2
@@ -2080,22 +2127,26 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX8-LABEL: add_i64_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
-; GFX8-NEXT: s_mul_i32 s6, s3, s8
+; GFX8-NEXT: s_mul_i32 s8, s2, s6
+; GFX8-NEXT: s_mul_i32 s6, s3, s6
+; GFX8-NEXT: s_add_u32 s9, s7, s6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GFX8-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB5_2:
@@ -2104,7 +2155,6 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u32 v3, s3, v2
; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
; GFX8-NEXT: s_mov_b32 s7, 0xf000
@@ -2118,23 +2168,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX9-LABEL: add_i64_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s7, s3, s6
-; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6
-; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_i32 s6, s2, s6
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_mul_i32 s8, s2, s6
+; GFX9-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX9-NEXT: s_mul_i32 s6, s3, s6
+; GFX9-NEXT: s_add_u32 s9, s7, s6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB5_2:
@@ -2156,23 +2206,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1064-LABEL: add_i64_uniform:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s7, s3, s6
-; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1064-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1064-NEXT: s_mul_i32 s8, s3, s6
; GFX1064-NEXT: s_mul_i32 s6, s2, s6
-; GFX1064-NEXT: s_add_i32 s8, s8, s7
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064-NEXT: v_mov_b32_e32 v1, s8
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
; GFX1064-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -2192,21 +2242,21 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1032-LABEL: add_i64_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s6, s3, s5
; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1032-NEXT: s_mul_i32 s5, s2, s5
-; GFX1032-NEXT: s_add_i32 s7, s7, s6
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s8, s3, s5
+; GFX1032-NEXT: s_mul_i32 s6, s2, s5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -2227,24 +2277,25 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1164-LABEL: add_i64_uniform:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s7, s3, s6
-; GFX1164-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1164-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1164-NEXT: s_mul_i32 s8, s3, s6
; GFX1164-NEXT: s_mul_i32 s6, s2, s6
-; GFX1164-NEXT: s_add_i32 s8, s8, s7
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164-NEXT: v_mov_b32_e32 v1, s8
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
; GFX1164-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -2265,23 +2316,22 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1132-LABEL: add_i64_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s6, s3, s5
; GFX1132-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1132-NEXT: s_mul_i32 s5, s2, s5
-; GFX1132-NEXT: s_add_i32 s7, s7, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mul_i32 s8, s3, s5
+; GFX1132-NEXT: s_mul_i32 s6, s2, s5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -2302,21 +2352,23 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1364-LABEL: add_i64_uniform:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_mov_b32 s9, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB5_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[6:7], s[2:3], s[8:9]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1364-NEXT: s_mul_i32 s8, s3, s6
+; GFX1364-NEXT: s_mul_i32 s6, s2, s6
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
; GFX1364-NEXT: v_mov_b32_e32 v1, s7
; GFX1364-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
@@ -2338,26 +2390,27 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, i64 %additive)
; GFX1332-LABEL: add_i64_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s5, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s7, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB5_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[4:5], s[2:3], s[4:5]
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
-; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mul_hi_u32 s7, s2, s5
+; GFX1332-NEXT: s_mul_i32 s8, s3, s5
+; GFX1332-NEXT: s_mul_i32 s6, s2, s5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1332-NEXT: v_mov_b32_e32 v1, s7
; GFX1332-NEXT: ds_add_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB5_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX1332-NEXT: v_readfirstlane_b32 s5, v1
; GFX1332-NEXT: v_readfirstlane_b32 s4, v0
; GFX1332-NEXT: s_wait_kmcnt 0x0
@@ -3433,33 +3486,28 @@ entry:
define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX7LESS_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: s_nop 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
; GFX7LESS_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
@@ -3468,30 +3516,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX8_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX8_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX8_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
; GFX8_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
@@ -3500,30 +3546,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX9_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX9_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX9_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
-; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX9_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB7_4:
@@ -3531,30 +3575,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1064_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1064_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
; GFX1064_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1064_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3563,29 +3605,27 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1032_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s5, v2, s3
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s2, s3
; GFX1032_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s2, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1032_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3594,33 +3634,31 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1164_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s4, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s4
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[2:3], s4
; GFX1164_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[2:3], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1164_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3629,31 +3667,28 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1132_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s5, v2, s3
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s2, s3
; GFX1132_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s2, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB7_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
-; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
; GFX1132_ITERATIVE-NEXT: ds_add_u64 v2, v[0:1]
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
@@ -3674,46 +3709,52 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX8_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8_DPP-NEXT: v_add_u32_e32 v2, vcc, v2, v4
-; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8_DPP-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX8_DPP-NEXT: v_readlane_b32 s3, v1, 63
-; GFX8_DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX8_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX8_DPP-NEXT: s_nop 0
+; GFX8_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_e32 v1, vcc, v1, v3
+; GFX8_DPP-NEXT: v_addc_u32_e32 v2, vcc, v2, v4, vcc
+; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX8_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
@@ -3734,46 +3775,52 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v6
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9_DPP-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
-; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9_DPP-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX9_DPP-NEXT: v_readlane_b32 s3, v1, 63
-; GFX9_DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX9_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX9_DPP-NEXT: s_nop 0
+; GFX9_DPP-NEXT: v_mov_b32_dpp v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v4 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9_DPP-NEXT: v_addc_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX9_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
@@ -3790,55 +3837,64 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1064_DPP-LABEL: add_i64_varying_nouse:
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s[0:1]
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v2, v1
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v3, v4, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v4
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v3, vcc
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v6
-; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v5, vcc
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
+; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
+; GFX1064_DPP-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1064_DPP-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(1)
; GFX1064_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc, v2, v4, vcc
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v2, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s4, v1, 32
-; GFX1064_DPP-NEXT: v_readlane_b32 s5, v2, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1064_DPP-NEXT: v_readlane_b32 s3, v2, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_u32 s0, s3, s4
-; GFX1064_DPP-NEXT: s_addc_u32 s1, s2, s5
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
+; GFX1064_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s1
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s0
-; GFX1064_DPP-NEXT: ds_add_u64 v0, v[7:8]
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v7, s1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, s0
+; GFX1064_DPP-NEXT: ds_add_u64 v0, v[6:7]
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB7_2:
@@ -3846,47 +3902,51 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i64_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
-; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v3, 0, 0, s0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v2, v1
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v3, v4, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v6, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v6
-; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v3, v1, 0, 0
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
+; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s2
+; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
-; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
+; GFX1032_DPP-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v9
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v5
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_add_u64 v0, v[7:8]
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, s1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, s0
+; GFX1032_DPP-NEXT: ds_add_u64 v0, v[5:6]
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB7_2:
@@ -3897,60 +3957,85 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1164_DPP-NEXT: v_permlane64_b32 v3, v2
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1164_DPP-NEXT: v_permlane64_b32 v4, v1
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1164_DPP-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1164_DPP-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_va_vcc(0)
+; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0) depctr_va_vcc(0)
-; GFX1164_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s3, v2, 63
+; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v2
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v3
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_add_u64 v0, v[5:6]
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s1
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s0
+; GFX1164_DPP-NEXT: ds_add_u64 v0, v[6:7]
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB7_2:
@@ -3959,47 +4044,58 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1132_DPP-LABEL: add_i64_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s2
+; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
-; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v3
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(1)
+; GFX1132_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v5
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1132_DPP-NEXT: ; %bb.1:
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v5, s0
; GFX1132_DPP-NEXT: ds_add_u64 v0, v[5:6]
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
@@ -4010,57 +4106,81 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, 0, s[0:1]
-; GFX1364-NEXT: v_mov_b32_e32 v2, 0
-; GFX1364-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
-; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
+; GFX1364-NEXT: v_cndmask_b32_e64 v2, 0, 0, s[0:1]
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_mov_b32_e32 v2, 0
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v5, 32, v5
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v3, vcc, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_mov_b32_e32 v4, 0
-; GFX1364-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc
-; GFX1364-NEXT: v_add_co_u32_e64_dpp v2, vcc, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1364-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1364-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: v_add_co_u32 v2, vcc, v2, v3
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1364-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1364-NEXT: v_permlane64_b32 v3, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc
-; GFX1364-NEXT: v_permlane64_b32 v4, v1
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: v_mov_b32_e32 v3, v1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_e32 v4, v2
+; GFX1364-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
+; GFX1364-NEXT: ds_permute_b32 v3, v5, v1
+; GFX1364-NEXT: ds_permute_b32 v4, v5, v2
+; GFX1364-NEXT: s_wait_dscnt 0x1
+; GFX1364-NEXT: v_add_co_u32 v1, vcc, v1, v3
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: v_add_co_u32 v2, vcc, v2, v3
-; GFX1364-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
+; GFX1364-NEXT: v_readlane_b32 s3, v2, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: v_mov_b32_e32 v5, v2
-; GFX1364-NEXT: v_mov_b32_e32 v6, v3
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1364-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v6
; GFX1364-NEXT: s_cbranch_execz .LBB7_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_add_u64 v0, v[5:6]
+; GFX1364-NEXT: v_mov_b32_e32 v7, s1
+; GFX1364-NEXT: v_mov_b32_e32 v6, s0
+; GFX1364-NEXT: ds_add_u64 v0, v[6:7]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB7_2:
@@ -4069,46 +4189,58 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1332-LABEL: add_i64_varying_nouse:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1332-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
-; GFX1332-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v3, 0, v0, s0
-; GFX1332-NEXT: v_mov_b32_e32 v4, 0
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_mov_b32_dpp v2, v1 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1332-NEXT: v_mov_b32_e32 v2, 0
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_mov_b32_dpp v4, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v3, vcc_lo, v3, v3 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_mov_b32_e32 v4, 0
-; GFX1332-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v2, vcc_lo
-; GFX1332-NEXT: v_add_co_u32_e64_dpp v2, vcc_lo, v3, v3 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mov_b32_dpp v4, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1332-NEXT: v_permlanex16_b32 v3, v2, 0, 0
+; GFX1332-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
+; GFX1332-NEXT: v_dual_cndmask_b32 v1, 0, v0, s2 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_e32 v3, v1
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v4, vcc_lo
-; GFX1332-NEXT: v_add_co_u32 v2, vcc_lo, v2, v3
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v4, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mov_b32_dpp v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v4, v1, 0, 0
-; GFX1332-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
-; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
-; GFX1332-NEXT: v_mov_b32_e32 v6, v3
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v7
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: ds_swizzle_b32 v3, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x1
+; GFX1332-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
+; GFX1332-NEXT: s_wait_dscnt 0x0
+; GFX1332-NEXT: v_add_co_ci_u32_e64 v2, null, v2, v4, vcc_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_readlane_b32 s0, v1, 31
+; GFX1332-NEXT: v_readlane_b32 s1, v2, 31
+; GFX1332-NEXT: s_mov_b32 exec_lo, s2
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v5, exec_lo, 0
+; GFX1332-NEXT: v_mov_b32_e32 v0, 0
+; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v5
; GFX1332-NEXT: s_cbranch_execz .LBB7_2
; GFX1332-NEXT: ; %bb.1:
+; GFX1332-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v5, s0
; GFX1332-NEXT: ds_add_u64 v0, v[5:6]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
@@ -4124,15 +4256,15 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: sub_i32_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
@@ -4153,15 +4285,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB8_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -4182,15 +4314,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB8_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -4210,17 +4342,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: sub_i32_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB8_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: v_mov_b32_e32 v2, s2
; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
@@ -4240,16 +4372,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i32_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s1
; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -4269,19 +4401,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: sub_i32_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB8_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: v_mov_b32_e32 v2, s2
; GFX1164-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4301,17 +4433,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i32_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1132-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4331,19 +4463,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: sub_i32_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB8_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: v_mov_b32_e32 v2, s2
; GFX1364-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
@@ -4363,17 +4495,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: sub_i32_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB8_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s1, s1, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s1
; GFX1332-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1332-NEXT: s_wait_dscnt 0x0
@@ -4399,20 +4531,20 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive) {
; GFX7LESS-LABEL: sub_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_load_dword s2, s[4:5], 0xb
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
@@ -4420,7 +4552,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -4430,20 +4562,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -4451,7 +4583,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -4461,27 +4593,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB9_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -4491,20 +4623,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1064-LABEL: sub_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -4513,7 +4645,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1064-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4524,17 +4656,17 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1032-LABEL: sub_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mul_i32 s2, s0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: v_mov_b32_e32 v2, s2
; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -4554,22 +4686,22 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1164-LABEL: sub_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB9_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -4577,7 +4709,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1164-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4588,15 +4720,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1132-LABEL: sub_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB9_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mul_i32 s2, s0, s2
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -4618,22 +4750,22 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1364-LABEL: sub_i32_uniform:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_load_b32 s6, s[4:5], 0x2c nv
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: s_load_b32 s2, s[4:5], 0x2c nv
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB9_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_i32 s2, s6, s2
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mov_b32_e32 v2, s2
+; GFX1364-NEXT: s_mul_i32 s3, s2, s3
+; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: v_mov_b32_e32 v2, s3
; GFX1364-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
@@ -4641,7 +4773,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1364-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX1364-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1364-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1364-NEXT: v_readfirstlane_b32 s2, v1
; GFX1364-NEXT: s_mov_b32 s3, 0x31016000
; GFX1364-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -4652,15 +4784,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
; GFX1332-LABEL: sub_i32_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
-; GFX1332-NEXT: s_mov_b32 s2, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s1, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB9_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: s_mul_i32 s2, s0, s2
; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -5395,112 +5527,102 @@ entry:
define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.2:
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX7LESS_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX7LESS_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: .LBB11_4:
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
-; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX8_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX8_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX8_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8_ITERATIVE-NEXT: ; %bb.2:
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, -1
-; GFX8_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX8_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: .LBB11_4:
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
-; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX9_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX9_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
; GFX9_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9_ITERATIVE-NEXT: ; %bb.2:
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX9_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX9_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: .LBB11_4:
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; %bb.2:
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1064_ITERATIVE-NEXT: .LBB11_4:
@@ -5508,26 +5630,24 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s2, s1
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1032_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1032_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s1, s1, s2
+; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; %bb.2:
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s1, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s0
-; GFX1032_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1032_ITERATIVE-NEXT: .LBB11_4:
@@ -5535,30 +5655,28 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, 0
-; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s3
-; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s6
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164_ITERATIVE-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164_ITERATIVE-NEXT: s_add_i32 s2, s2, s4
+; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: ; %bb.2:
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1164_ITERATIVE-NEXT: .LBB11_4:
@@ -5566,28 +5684,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: sub_i32_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, 0
-; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s2
+; GFX1132_ITERATIVE-NEXT: s_bitset0_b32 s1, s2
; GFX1132_ITERATIVE-NEXT: s_add_i32 s0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s1, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: ; %bb.2:
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s1, exec_lo, s1
+; GFX1132_ITERATIVE-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1132_ITERATIVE-NEXT: ds_sub_u32 v0, v1
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
+; GFX1132_ITERATIVE-NEXT: ds_sub_u32 v1, v0
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
; GFX1132_ITERATIVE-NEXT: .LBB11_4:
@@ -5603,31 +5719,31 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX8_DPP-LABEL: sub_i32_varying_nouse:
; GFX8_DPP: ; %bb.0: ; %entry
-; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX8_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX8_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX8_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX8_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
-; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX8_DPP-NEXT: v_add_u32_dpp v1, vcc, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX8_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX8_DPP-NEXT: s_mov_b32 s0, s2
-; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX8_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX8_DPP-NEXT: ; %bb.1:
; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX8_DPP-NEXT: s_mov_b32 m0, -1
; GFX8_DPP-NEXT: ds_sub_u32 v2, v0
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5636,31 +5752,31 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX9_DPP-LABEL: sub_i32_varying_nouse:
; GFX9_DPP: ; %bb.0: ; %entry
-; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
-; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v3
+; GFX9_DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9_DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
; GFX9_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
-; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xc bank_mask:0xf
+; GFX9_DPP-NEXT: v_add_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX9_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9_DPP-NEXT: s_mov_b32 s0, s2
-; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX9_DPP-NEXT: ; %bb.1:
; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9_DPP-NEXT: ds_sub_u32 v2, v0
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_DPP-NEXT: .LBB11_2:
@@ -5670,27 +5786,34 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1064_DPP: ; %bb.0: ; %entry
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1064_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1064_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1064_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1064_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 0
-; GFX1064_DPP-NEXT: v_readlane_b32 s3, v1, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, exec_hi, v0
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064_DPP-NEXT: s_add_i32 s0, s2, s3
-; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
+; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v3, s0
-; GFX1064_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064_DPP-NEXT: ds_sub_u32 v4, v0
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl0_inv
; GFX1064_DPP-NEXT: .LBB11_2:
@@ -5700,21 +5823,24 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1032_DPP: ; %bb.0: ; %entry
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1032_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_DPP-NEXT: ds_sub_u32 v3, v0
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl0_inv
; GFX1032_DPP-NEXT: .LBB11_2:
@@ -5726,33 +5852,41 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1164_DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164_DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164_DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164_DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164_DPP-NEXT: ds_sub_u32 v4, v0
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl0_inv
; GFX1164_DPP-NEXT: .LBB11_2:
@@ -5765,24 +5899,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132_DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132_DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: ds_sub_u32 v0, v3
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1132_DPP-NEXT: ds_sub_u32 v3, v0
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl0_inv
; GFX1132_DPP-NEXT: .LBB11_2:
@@ -5794,33 +5930,40 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1364-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1364-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1364-NEXT: s_wait_dscnt 0x0
+; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_permlane64_b32 v2, v1
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1364-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1364-NEXT: v_readlane_b32 s2, v1, 63
; GFX1364-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1364-NEXT: v_mov_b32_e32 v0, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1364-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_mov_b32 s0, s2
+; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB11_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: ds_sub_u32 v0, v3
+; GFX1364-NEXT: v_mov_b32_e32 v0, s0
+; GFX1364-NEXT: v_mov_b32_e32 v4, 0
+; GFX1364-NEXT: ds_sub_u32 v4, v0
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB11_2:
@@ -5833,24 +5976,26 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:2 row_mask:0xf bank_mask:0xf bound_ctrl:1
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:4 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:8 row_mask:0xf bank_mask:0xf bound_ctrl:1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1332-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1332-NEXT: v_readlane_b32 s1, v1, 31
; GFX1332-NEXT: s_mov_b32 exec_lo, s0
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1332-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
-; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s0, s1
+; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB11_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: ds_sub_u32 v0, v3
+; GFX1332-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
+; GFX1332-NEXT: ds_sub_u32 v3, v0
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB11_2:
@@ -5864,20 +6009,25 @@ entry:
define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
; GFX7LESS-LABEL: sub_i64_constant:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s3, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mul_i32 s2, s2, 5
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS-NEXT: s_mul_i32 s6, s2, 5
+; GFX7LESS-NEXT: s_mul_i32 s2, s2, 0
+; GFX7LESS-NEXT: s_add_u32 s7, s3, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
; GFX7LESS-NEXT: s_mov_b32 m0, -1
-; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB12_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5897,20 +6047,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX8-LABEL: sub_i64_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_mul_hi_u32 v0, 5, s2
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB12_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: s_mul_i32 s6, s2, 5
+; GFX8-NEXT: s_mul_i32 s2, s2, 0
+; GFX8-NEXT: s_add_u32 s7, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB12_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5930,19 +6085,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX9-LABEL: sub_i64_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX9-NEXT: s_mul_i32 s6, s2, 5
+; GFX9-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX9-NEXT: s_mul_i32 s2, s2, 0
+; GFX9-NEXT: s_add_u32 s7, s3, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB12_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
@@ -5962,19 +6121,23 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1064-LABEL: sub_i64_constant:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1064-NEXT: s_mul_i32 s6, s2, 0
; GFX1064-NEXT: s_mul_i32 s2, s2, 5
+; GFX1064-NEXT: s_add_u32 s3, s3, s6
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s2
-; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v1, s3
+; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB12_2:
@@ -5995,18 +6158,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
-; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1032-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1032-NEXT: s_mul_i32 s6, s1, 0
+; GFX1032-NEXT: s_mul_i32 s2, s1, 5
+; GFX1032-NEXT: s_add_u32 s3, s3, s6
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s2
+; GFX1032-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB12_2:
@@ -6027,21 +6194,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1164-LABEL: sub_i64_constant:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1164-NEXT: s_mul_i32 s6, s2, 0
; GFX1164-NEXT: s_mul_i32 s2, s2, 5
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_add_u32 s3, s3, s6
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s2
-; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1164-NEXT: v_mov_b32_e32 v1, s3
+; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB12_2:
@@ -6062,20 +6233,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
-; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1132-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1132-NEXT: s_mul_i32 s6, s1, 0
+; GFX1132-NEXT: s_mul_i32 s2, s1, 5
+; GFX1132-NEXT: s_add_u32 s3, s3, s6
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1132-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB12_2:
@@ -6096,21 +6269,25 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1364-LABEL: sub_i64_constant:
; GFX1364: ; %bb.0: ; %entry
-; GFX1364-NEXT: s_mov_b64 s[2:3], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[0:1], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s3, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX1364-NEXT: s_mov_b64 s[0:1], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB12_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1364-NEXT: v_mov_b32_e32 v1, 0
+; GFX1364-NEXT: s_mul_hi_u32 s3, 5, s2
+; GFX1364-NEXT: s_mul_i32 s6, s2, 0
; GFX1364-NEXT: s_mul_i32 s2, s2, 5
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s2
-; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1364-NEXT: v_mov_b32_e32 v1, s3
+; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1364-NEXT: s_wait_dscnt 0x0
; GFX1364-NEXT: global_inv scope:SCOPE_SE
; GFX1364-NEXT: .LBB12_2:
@@ -6131,20 +6308,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1332-LABEL: sub_i64_constant:
; GFX1332: ; %bb.0: ; %entry
-; GFX1332-NEXT: s_mov_b32 s1, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s0, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX1332-NEXT: s_mov_b32 s0, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB12_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX1332-NEXT: v_mov_b32_e32 v1, 0
-; GFX1332-NEXT: s_mul_i32 s1, s1, 5
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_mov_b32_e32 v0, s1
-; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
+; GFX1332-NEXT: s_mul_hi_u32 s3, 5, s1
+; GFX1332-NEXT: s_mul_i32 s6, s1, 0
+; GFX1332-NEXT: s_mul_i32 s2, s1, 5
+; GFX1332-NEXT: s_add_co_u32 s3, s3, s6
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s2
+; GFX1332-NEXT: v_mov_b32_e32 v1, s3
+; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB12_2:
@@ -6172,29 +6351,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX7LESS-LABEL: sub_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[4:5], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX7LESS-NEXT: s_mul_i32 s7, s3, s6
-; GFX7LESS-NEXT: s_mul_i32 s6, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s8, s2, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s3, s6
+; GFX7LESS-NEXT: s_add_u32 s9, s7, s6
; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s7, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s9
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB13_2:
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s4, s0
; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
; GFX7LESS-NEXT: v_mul_lo_u32 v1, s3, v2
@@ -6214,27 +6394,30 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX8-LABEL: sub_i64_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: s_mov_b64 s[4:5], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB13_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[6:7], s2, v0, 0
-; GFX8-NEXT: s_mul_i32 s6, s3, s8
+; GFX8-NEXT: s_mul_i32 s8, s2, s6
+; GFX8-NEXT: s_mul_i32 s6, s3, s6
+; GFX8-NEXT: s_add_u32 s9, s7, s6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s6, v1
; GFX8-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB13_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s4, s0
; GFX8-NEXT: s_mov_b32 s5, s1
; GFX8-NEXT: v_mul_lo_u32 v4, s3, v2
@@ -6253,23 +6436,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX9-LABEL: sub_i64_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s7, s3, s6
-; GFX9-NEXT: s_mul_hi_u32 s8, s2, s6
-; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_i32 s6, s2, s6
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_mul_i32 s8, s2, s6
+; GFX9-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX9-NEXT: s_mul_i32 s6, s3, s6
+; GFX9-NEXT: s_add_u32 s9, s7, s6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB13_2:
@@ -6292,23 +6475,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1064-LABEL: sub_i64_uniform:
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], exec
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s7, s3, s6
-; GFX1064-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1064-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1064-NEXT: s_mul_i32 s8, s3, s6
; GFX1064-NEXT: s_mul_i32 s6, s2, s6
-; GFX1064-NEXT: s_add_i32 s8, s8, s7
+; GFX1064-NEXT: s_add_u32 s7, s7, s8
+; GFX1064-NEXT: v_mov_b32_e32 v3, 0
; GFX1064-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064-NEXT: v_mov_b32_e32 v1, s8
+; GFX1064-NEXT: v_mov_b32_e32 v1, s7
; GFX1064-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
@@ -6330,21 +6513,21 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1032-LABEL: sub_i64_uniform:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s4, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s4
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s6, s3, s5
; GFX1032-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1032-NEXT: s_mul_i32 s5, s2, s5
-; GFX1032-NEXT: s_add_i32 s7, s7, s6
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032-NEXT: s_mul_i32 s8, s3, s5
+; GFX1032-NEXT: s_mul_i32 s6, s2, s5
+; GFX1032-NEXT: s_add_u32 s7, s7, s8
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, s7
; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
@@ -6367,24 +6550,25 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1164-LABEL: sub_i64_uniform:
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[4:5], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s7, s3, s6
-; GFX1164-NEXT: s_mul_hi_u32 s8, s2, s6
+; GFX1164-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1164-NEXT: s_mul_i32 s8, s3, s6
; GFX1164-NEXT: s_mul_i32 s6, s2, s6
-; GFX1164-NEXT: s_add_i32 s8, s8, s7
+; GFX1164-NEXT: s_add_u32 s7, s7, s8
+; GFX1164-NEXT: v_mov_b32_e32 v3, 0
; GFX1164-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164-NEXT: v_mov_b32_e32 v1, s8
+; GFX1164-NEXT: v_mov_b32_e32 v1, s7
; GFX1164-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
@@ -6406,23 +6590,22 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1132-LABEL: sub_i64_uniform:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1132-NEXT: s_mov_b32 s4, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s6, s3, s5
; GFX1132-NEXT: s_mul_hi_u32 s7, s2, s5
-; GFX1132-NEXT: s_mul_i32 s5, s2, s5
-; GFX1132-NEXT: s_add_i32 s7, s7, s6
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, s7
+; GFX1132-NEXT: s_mul_i32 s8, s3, s5
+; GFX1132-NEXT: s_mul_i32 s6, s2, s5
+; GFX1132-NEXT: s_add_u32 s7, s7, s8
+; GFX1132-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1132-NEXT: v_mov_b32_e32 v1, s7
; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -6444,21 +6627,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1364-LABEL: sub_i64_uniform:
; GFX1364: ; %bb.0: ; %entry
; GFX1364-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1364-NEXT: s_mov_b64 s[6:7], exec
-; GFX1364-NEXT: s_mov_b32 s9, 0
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[4:5], exec
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; GFX1364-NEXT: s_mov_b64 s[4:5], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v0
; GFX1364-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1364-NEXT: s_cbranch_execz .LBB13_2
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s8, s[6:7]
-; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: s_wait_kmcnt 0x0
-; GFX1364-NEXT: s_mul_u64 s[6:7], s[2:3], s[8:9]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1364-NEXT: s_mul_hi_u32 s7, s2, s6
+; GFX1364-NEXT: s_mul_i32 s8, s3, s6
+; GFX1364-NEXT: s_mul_i32 s6, s2, s6
+; GFX1364-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1364-NEXT: v_mov_b32_e32 v3, 0
; GFX1364-NEXT: v_mov_b32_e32 v0, s6
; GFX1364-NEXT: v_mov_b32_e32 v1, s7
; GFX1364-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
@@ -6482,26 +6667,27 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, i64 %subitive)
; GFX1332-LABEL: sub_i64_uniform:
; GFX1332: ; %bb.0: ; %entry
; GFX1332-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1332-NEXT: s_mov_b32 s6, exec_lo
-; GFX1332-NEXT: s_mov_b32 s5, 0
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1332-NEXT: s_mov_b32 s7, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s5, s4
+; GFX1332-NEXT: s_mov_b32 s4, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1332-NEXT: s_cbranch_execz .LBB13_2
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1332-NEXT: s_wait_kmcnt 0x0
-; GFX1332-NEXT: s_mul_u64 s[4:5], s[2:3], s[4:5]
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
-; GFX1332-NEXT: v_mov_b32_e32 v1, s5
+; GFX1332-NEXT: s_mul_hi_u32 s7, s2, s5
+; GFX1332-NEXT: s_mul_i32 s8, s3, s5
+; GFX1332-NEXT: s_mul_i32 s6, s2, s5
+; GFX1332-NEXT: s_add_co_u32 s7, s7, s8
+; GFX1332-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1332-NEXT: v_mov_b32_e32 v1, s7
; GFX1332-NEXT: ds_sub_rtn_u64 v[0:1], v3, v[0:1]
; GFX1332-NEXT: s_wait_dscnt 0x0
; GFX1332-NEXT: global_inv scope:SCOPE_SE
; GFX1332-NEXT: .LBB13_2:
-; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s7
+; GFX1332-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX1332-NEXT: s_wait_kmcnt 0x0
; GFX1332-NEXT: v_mad_co_u64_u32 v[3:4], null, s2, v2, 0
; GFX1332-NEXT: v_readfirstlane_b32 s2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
index c869e47a19987..7cdf2562336a9 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
@@ -23,8 +23,7 @@ define amdgpu_cs void @atomic_add_i32_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -35,14 +34,14 @@ define amdgpu_cs void @atomic_add_i32_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
@@ -63,9 +62,7 @@ define amdgpu_cs void @atomic_add_i32_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 0, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -101,7 +98,7 @@ define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 1, i32 1)
; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR: [[BB8]]:
@@ -112,15 +109,15 @@ define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add_i64_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB2_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: v_mov_b32_e32 v2, 0
@@ -142,8 +139,7 @@ define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 0, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -174,22 +170,21 @@ define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_add_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
-; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
+; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP10]], %[[TMP9]] ]
; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
; IR-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP5]]
; IR-NEXT: call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> [[ARG]], <4 x i32> [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
@@ -197,15 +192,15 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB4_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
@@ -237,8 +232,7 @@ define amdgpu_cs void @atomic_sub_i32_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -249,14 +243,14 @@ define amdgpu_cs void @atomic_sub_i32_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB5_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
@@ -277,9 +271,7 @@ define amdgpu_cs void @atomic_sub_i32_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 0, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -315,7 +307,7 @@ define amdgpu_cs void @atomic_sub_i64_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 1, i32 1)
; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
; IR: [[BB8]]:
@@ -326,15 +318,15 @@ define amdgpu_cs void @atomic_sub_i64_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub_i64_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB7_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: v_mov_b32_e32 v2, 0
@@ -356,8 +348,7 @@ define amdgpu_cs void @atomic_sub_i64_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.wave.reduce.add.i64(i64 0, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -388,22 +379,21 @@ define amdgpu_cs void @atomic_sub_i64_constant_0(<4 x i32> inreg %arg) {
define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_sub_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
-; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
+; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP10]], %[[TMP9]] ]
; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
; IR-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], [[TMP5]]
; IR-NEXT: call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> [[ARG]], <4 x i32> [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
@@ -411,15 +401,15 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB9_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
@@ -451,9 +441,7 @@ define amdgpu_cs void @atomic_xor_i32_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -464,14 +452,14 @@ define amdgpu_cs void @atomic_xor_i32_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB10_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
@@ -493,10 +481,7 @@ define amdgpu_cs void @atomic_xor_i32_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
-; IR-NEXT: [[TMP9:%.*]] = mul i32 0, [[TMP8]]
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 0, i32 1)
; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
; IR: [[BB11]]:
@@ -532,9 +517,7 @@ define amdgpu_cs void @atomic_xor_i64_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
-; IR-NEXT: [[TMP8:%.*]] = mul i64 0, [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i64 @llvm.amdgcn.wave.reduce.xor.i64(i64 0, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -572,8 +555,7 @@ define amdgpu_cs void @atomic_xor_i64_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.wave.reduce.xor.i64(i64 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -584,16 +566,16 @@ define amdgpu_cs void @atomic_xor_i64_constant_1(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor_i64_constant_1:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB13_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], 1
+; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: v_mov_b32_e32 v2, 0
@@ -608,23 +590,21 @@ define amdgpu_cs void @atomic_xor_i64_constant_1(<4 x i32> inreg %arg) {
define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_xor_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
-; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP11]], %[[TMP10]] ]
+; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP11]], %[[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = and i32 [[TMP5]], 1
; IR-NEXT: [[TMP16:%.*]] = xor i32 [[TMP14]], [[TMP15]]
@@ -633,15 +613,15 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
;
; GCN-LABEL: atomic_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB14_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
@@ -675,8 +655,7 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -687,14 +666,14 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_add:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB15_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
@@ -708,22 +687,21 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_ptr_add_and_format(
; IR-SAME: ptr addrspace(8) inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
-; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
+; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP10]], %[[TMP9]] ]
; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
; IR-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP5]]
; IR-NEXT: [[ARG_INT:%.*]] = ptrtoint ptr addrspace(8) [[ARG]] to i128
@@ -733,15 +711,15 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB16_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
@@ -775,8 +753,7 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
; IR: [[BB9]]:
@@ -787,14 +764,14 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_sub:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB17_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
@@ -808,22 +785,21 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_ptr_sub_and_format(
; IR-SAME: ptr addrspace(8) inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 1, i32 1)
; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
; IR: [[TMP9]]:
; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 [[TMP7]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB11]]
; IR: [[BB11]]:
-; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
+; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP10]], %[[TMP9]] ]
; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
; IR-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], [[TMP5]]
; IR-NEXT: [[ARG_INT:%.*]] = ptrtoint ptr addrspace(8) [[ARG]] to i128
@@ -833,15 +809,15 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_sub_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB18_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
@@ -875,9 +851,7 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
; IR: [[BB10]]:
@@ -888,14 +862,14 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_xor:
; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB19_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
@@ -910,23 +884,21 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_ptr_xor_and_format(
; IR-SAME: ptr addrspace(8) inreg [[ARG:%.*]]) {
-; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[_ENTRY:.*]]:
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.xor.i32(i32 1, i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label %[[TMP10:.*]], label %[[BB12:.*]]
; IR: [[TMP10]]:
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 [[TMP8]], ptr addrspace(8) [[ARG]], i32 0, i32 0, i32 0, i32 0)
; IR-NEXT: br label %[[BB12]]
; IR: [[BB12]]:
-; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP11]], %[[TMP10]] ]
+; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, %[[_ENTRY]] ], [ [[TMP11]], %[[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = and i32 [[TMP5]], 1
; IR-NEXT: [[TMP16:%.*]] = xor i32 [[TMP14]], [[TMP15]]
@@ -937,15 +909,15 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
;
; GCN-LABEL: atomic_ptr_xor_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB20_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 4c87e6936e314..d3d7e8131a0c1 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -23,15 +23,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX7-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX7-NEXT: s_cbranch_execz .LBB0_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_mov_b64 s[12:13], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s12, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s13, v0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX7-NEXT: s_mov_b64 s[10:11], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7-NEXT: s_cbranch_execz .LBB0_3
; GFX7-NEXT: ; %bb.2:
-; GFX7-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX7-NEXT: s_mul_i32 s12, s12, 5
; GFX7-NEXT: v_mov_b32_e32 v1, s12
; GFX7-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -60,15 +60,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX89-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX89-NEXT: s_cbranch_execz .LBB0_4
; GFX89-NEXT: ; %bb.1:
-; GFX89-NEXT: s_mov_b64 s[12:13], exec
-; GFX89-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX89-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX89-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX89-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX89-NEXT: s_mov_b64 s[10:11], exec
+; GFX89-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
; GFX89-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX89-NEXT: ; implicit-def: $vgpr1
; GFX89-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX89-NEXT: s_cbranch_execz .LBB0_3
; GFX89-NEXT: ; %bb.2:
-; GFX89-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX89-NEXT: s_mul_i32 s12, s12, 5
; GFX89-NEXT: v_mov_b32_e32 v1, s12
; GFX89-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -97,15 +97,15 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1064-NEXT: s_cbranch_execz .LBB0_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_mov_b64 s[12:13], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1064-NEXT: s_mov_b64 s[10:11], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.2:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
; GFX1064-NEXT: s_mul_i32 s12, s12, 5
; GFX1064-NEXT: v_mov_b32_e32 v1, s12
; GFX1064-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -135,14 +135,14 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1032-NEXT: s_and_saveexec_b32 s8, s9
; GFX1032-NEXT: s_cbranch_execz .LBB0_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_mov_b32 s10, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s9, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s9
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.2:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s10
; GFX1032-NEXT: s_mul_i32 s10, s10, 5
; GFX1032-NEXT: v_mov_b32_e32 v1, s10
; GFX1032-NEXT: buffer_atomic_add v1, off, s[4:7], 0 glc
@@ -173,18 +173,19 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1164-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1164-NEXT: s_cbranch_execz .LBB0_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_mov_b64 s[12:13], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[10:11], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1164-NEXT: s_mov_b64 s[10:11], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-NEXT: ; %bb.2:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: s_mul_i32 s12, s12, 5
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: v_mov_b32_e32 v1, s12
; GFX1164-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], 0 glc
; GFX1164-NEXT: .LBB0_3:
@@ -216,17 +217,17 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1132-NEXT: s_and_saveexec_b32 s8, s9
; GFX1132-NEXT: s_cbranch_execz .LBB0_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_mov_b32 s10, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s9, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s9
+; GFX1132-NEXT: s_mov_b32 s9, exec_lo
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-NEXT: ; %bb.2:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s10
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132-NEXT: s_mul_i32 s10, s10, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: v_mov_b32_e32 v1, s10
; GFX1132-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], 0 glc
; GFX1132-NEXT: .LBB0_3:
@@ -258,18 +259,19 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1364-NEXT: s_and_saveexec_b64 s[8:9], s[10:11]
; GFX1364-NEXT: s_cbranch_execz .LBB0_4
; GFX1364-NEXT: ; %bb.1:
-; GFX1364-NEXT: s_mov_b64 s[12:13], exec
+; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1364-NEXT: s_mov_b64 s[10:11], exec
-; GFX1364-NEXT: v_mbcnt_lo_u32_b32 v0, s12, 0
; GFX1364-NEXT: ; implicit-def: $vgpr1
-; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, s13, v0
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1364-NEXT: s_bcnt1_i32_b64 s12, s[10:11]
+; GFX1364-NEXT: s_mov_b64 s[10:11], exec
+; GFX1364-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1364-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1364-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1364-NEXT: s_cbranch_execz .LBB0_3
; GFX1364-NEXT: ; %bb.2:
-; GFX1364-NEXT: s_bcnt1_i32_b64 s12, s[12:13]
-; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1364-NEXT: s_mul_i32 s12, s12, 5
+; GFX1364-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1364-NEXT: v_mov_b32_e32 v1, s12
; GFX1364-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], null th:TH_ATOMIC_RETURN
; GFX1364-NEXT: .LBB0_3:
@@ -301,17 +303,17 @@ define amdgpu_ps void @add_i32_constant(ptr addrspace(8) inreg %out, ptr addrspa
; GFX1332-NEXT: s_and_saveexec_b32 s8, s9
; GFX1332-NEXT: s_cbranch_execz .LBB0_4
; GFX1332-NEXT: ; %bb.1:
-; GFX1332-NEXT: s_mov_b32 s10, exec_lo
+; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1332-NEXT: s_mov_b32 s9, exec_lo
-; GFX1332-NEXT: v_mbcnt_lo_u32_b32 v0, s10, 0
; GFX1332-NEXT: ; implicit-def: $vgpr1
-; GFX1332-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1332-NEXT: s_bcnt1_i32_b32 s10, s9
+; GFX1332-NEXT: s_mov_b32 s9, exec_lo
; GFX1332-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1332-NEXT: s_cbranch_execz .LBB0_3
; GFX1332-NEXT: ; %bb.2:
-; GFX1332-NEXT: s_bcnt1_i32_b32 s10, s10
-; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1332-NEXT: s_mul_i32 s10, s10, 5
+; GFX1332-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1332-NEXT: v_mov_b32_e32 v1, s10
; GFX1332-NEXT: buffer_atomic_add_u32 v1, off, s[4:7], null th:TH_ATOMIC_RETURN
; GFX1332-NEXT: .LBB0_3:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 68defb3528267..f09555fde5d32 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -20,16 +20,16 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -48,16 +48,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -76,16 +76,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -103,16 +103,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -131,15 +131,15 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -158,19 +158,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -188,18 +189,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
@@ -217,20 +218,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -249,19 +250,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -279,19 +279,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -309,18 +310,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -344,26 +345,26 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -374,26 +375,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -404,26 +405,26 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -433,29 +434,30 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -463,15 +465,15 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -491,31 +493,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -523,16 +526,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -552,33 +555,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -586,20 +588,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -616,31 +617,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -648,16 +650,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1284,16 +1286,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB4_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
@@ -1313,16 +1315,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -1342,16 +1344,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1370,16 +1372,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB4_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
@@ -1399,15 +1401,15 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB4_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
@@ -1427,19 +1429,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB4_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1458,18 +1461,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB4_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
@@ -1488,20 +1491,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB4_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1521,19 +1524,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB4_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1552,19 +1554,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB4_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1583,18 +1586,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB4_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_mov_b32_e32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
@@ -1618,26 +1621,26 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX6-NEXT: .LBB5_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -1648,26 +1651,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX8-NEXT: .LBB5_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -1678,26 +1681,26 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX9-NEXT: .LBB5_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1707,27 +1710,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
; GFX10W64-NEXT: .LBB5_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1738,15 +1741,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
@@ -1766,28 +1769,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
; GFX11W64-NEXT: .LBB5_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1799,16 +1803,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1829,29 +1833,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB5_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1864,20 +1868,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_mov_b32_e32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB5_2:
@@ -1896,28 +1899,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB5_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1929,16 +1933,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index 0a1e47d75d82d..83d8aa12d54aa 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -20,16 +20,16 @@ declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub(i32, ptr addrspace(8), i32
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: add_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: v_mov_b32_e32 v2, 0
@@ -49,16 +49,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: add_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, 0
@@ -78,16 +78,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -106,18 +106,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: add_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB0_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W64-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
@@ -135,17 +135,17 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: add_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB0_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
@@ -163,20 +163,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: add_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB0_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
@@ -194,20 +194,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: add_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB0_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB0_2:
@@ -224,21 +223,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: add_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB0_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -257,19 +255,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: add_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB0_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -287,20 +284,20 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: add_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB0_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -318,18 +315,18 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: add_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB0_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -353,27 +350,27 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %additive) {
; GFX6-LABEL: add_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB1_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: v_mov_b32_e32 v2, 0
; GFX6-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX6-NEXT: .LBB1_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -384,27 +381,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX8-NEXT: .LBB1_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_add_u32_e32 v2, vcc, s2, v0
@@ -415,27 +412,27 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX9-NEXT: .LBB1_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -445,30 +442,31 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: add_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB1_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX10W64-NEXT: .LBB1_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
-; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10W64-NEXT: s_mov_b32 null, 0
+; GFX10W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX10W64-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10W64-NEXT: s_endpgm
@@ -476,18 +474,18 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: add_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB1_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
; GFX10W32-NEXT: buffer_atomic_add v1, v2, s[8:11], 0 idxen glc
; GFX10W32-NEXT: .LBB1_2:
@@ -505,32 +503,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: add_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB1_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W64-NEXT: .LBB1_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
-; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX11W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
+; GFX11W64-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX11W64-NEXT: v_mov_b32_e32 v0, 0
; GFX11W64-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11W64-NEXT: s_endpgm
@@ -538,21 +536,20 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: add_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB1_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB1_2:
; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
@@ -568,34 +565,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: add_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB1_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB1_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_loadcnt 0x0
-; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX12W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
; GFX12W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12W64-NEXT: s_endpgm
@@ -603,20 +598,19 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: add_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB1_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12W32-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB1_2:
@@ -633,32 +627,32 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: add_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB1_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_add_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB1_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_loadcnt 0x0
-; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
+; GFX13W64-NEXT: v_readfirstlane_b32 s4, v1
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s6, v0, s[2:3]
+; GFX13W64-NEXT: v_mad_co_u64_u32 v[0:1], null, s2, v0, s[4:5]
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
; GFX13W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX13W64-NEXT: s_endpgm
@@ -666,16 +660,16 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: add_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB1_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1465,16 +1459,16 @@ entry:
define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX6-LABEL: sub_i32_constant:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB5_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_mul_i32 s2, s2, 5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: v_mov_b32_e32 v2, 0
@@ -1495,16 +1489,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX8-LABEL: sub_i32_constant:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_mul_i32 s2, s2, 5
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_mov_b32_e32 v2, 0
@@ -1525,16 +1519,16 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-LABEL: sub_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1554,18 +1548,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W64-LABEL: sub_i32_constant:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB5_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_mul_i32 s2, s2, 5
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W64-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
@@ -1584,17 +1578,17 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10W32-LABEL: sub_i32_constant:
; GFX10W32: ; %bb.0: ; %entry
-; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s0, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB5_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s1
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
@@ -1613,20 +1607,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W64-LABEL: sub_i32_constant:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB5_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
@@ -1645,20 +1639,19 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11W32-LABEL: sub_i32_constant:
; GFX11W32: ; %bb.0: ; %entry
-; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11W32-NEXT: s_mov_b32 s0, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB5_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_mul_i32 s1, s1, 5
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s1
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB5_2:
@@ -1676,21 +1669,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W64-LABEL: sub_i32_constant:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB5_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
; GFX12W64-NEXT: s_wait_kmcnt 0x0
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1710,19 +1702,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12W32-LABEL: sub_i32_constant:
; GFX12W32: ; %bb.0: ; %entry
-; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12W32-NEXT: s_mov_b32 s0, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB5_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s1, s1, 5
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX12W32-NEXT: s_wait_kmcnt 0x0
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1741,20 +1732,20 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W64-LABEL: sub_i32_constant:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB5_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_mul_i32 s2, s2, 5
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
; GFX13W64-NEXT: s_wait_kmcnt 0x0
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1773,18 +1764,18 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX13W32-LABEL: sub_i32_constant:
; GFX13W32: ; %bb.0: ; %entry
-; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX13W32-NEXT: s_mov_b32 s0, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB5_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX13W32-NEXT: s_mul_i32 s1, s1, 5
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX13W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s1
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
@@ -1808,27 +1799,27 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(8) %inout, i32 %subitive) {
; GFX6-LABEL: sub_i32_uniform:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x11
-; GFX6-NEXT: s_mov_b64 s[2:3], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_load_dword s2, s[4:5], 0x11
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB6_2
; GFX6-NEXT: ; %bb.1:
; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xd
-; GFX6-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s2, s6, s2
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
+; GFX6-NEXT: s_mul_i32 s3, s2, s3
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: v_mov_b32_e32 v2, 0
; GFX6-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX6-NEXT: .LBB6_2:
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_readfirstlane_b32 s4, v1
; GFX6-NEXT: s_mov_b32 s3, 0xf000
@@ -1839,27 +1830,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB6_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX8-NEXT: .LBB6_2:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v1
; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s2, v0
@@ -1870,27 +1861,27 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB6_2
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX9-NEXT: .LBB6_2:
; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
@@ -1900,28 +1891,28 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10W64-LABEL: sub_i32_uniform:
; GFX10W64: ; %bb.0: ; %entry
-; GFX10W64-NEXT: s_load_dword s6, s[4:5], 0x44
-; GFX10W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX10W64-NEXT: s_load_dword s2, s[4:5], 0x44
+; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W64-NEXT: s_mov_b64 s[0:1], exec
; GFX10W64-NEXT: ; implicit-def: $vgpr1
-; GFX10W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX10W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX10W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX10W64-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX10W64-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX10W64-NEXT: s_cbranch_execz .LBB6_2
; GFX10W64-NEXT: ; %bb.1:
; GFX10W64-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX10W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX10W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX10W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX10W64-NEXT: v_mov_b32_e32 v1, s3
; GFX10W64-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX10W64-NEXT: .LBB6_2:
; GFX10W64-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX10W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX10W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX10W64-NEXT: s_waitcnt vmcnt(0)
; GFX10W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX10W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1932,18 +1923,18 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX10W32-LABEL: sub_i32_uniform:
; GFX10W32: ; %bb.0: ; %entry
; GFX10W32-NEXT: s_load_dword s0, s[4:5], 0x44
-; GFX10W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10W32-NEXT: s_mov_b32 s1, exec_lo
; GFX10W32-NEXT: ; implicit-def: $vgpr1
-; GFX10W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s1
; GFX10W32-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10W32-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10W32-NEXT: s_cbranch_execz .LBB6_2
; GFX10W32-NEXT: ; %bb.1:
; GFX10W32-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
-; GFX10W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX10W32-NEXT: s_mul_i32 s2, s0, s2
+; GFX10W32-NEXT: v_mov_b32_e32 v2, 0
; GFX10W32-NEXT: v_mov_b32_e32 v1, s2
; GFX10W32-NEXT: buffer_atomic_sub v1, v2, s[8:11], 0 idxen glc
; GFX10W32-NEXT: .LBB6_2:
@@ -1961,29 +1952,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11W64-LABEL: sub_i32_uniform:
; GFX11W64: ; %bb.0: ; %entry
-; GFX11W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX11W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX11W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX11W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W64-NEXT: ; implicit-def: $vgpr1
-; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX11W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX11W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX11W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W64-NEXT: s_cbranch_execz .LBB6_2
; GFX11W64-NEXT: ; %bb.1:
; GFX11W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX11W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX11W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX11W64-NEXT: v_mov_b32_e32 v1, s3
; GFX11W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W64-NEXT: .LBB6_2:
; GFX11W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX11W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX11W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX11W64-NEXT: s_waitcnt vmcnt(0)
; GFX11W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX11W64-NEXT: v_mov_b32_e32 v1, 0
@@ -1995,21 +1986,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX11W32-LABEL: sub_i32_uniform:
; GFX11W32: ; %bb.0: ; %entry
; GFX11W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX11W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX11W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11W32-NEXT: ; implicit-def: $vgpr1
-; GFX11W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX11W32-NEXT: s_mov_b32 s1, exec_lo
; GFX11W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11W32-NEXT: s_cbranch_execz .LBB6_2
; GFX11W32-NEXT: ; %bb.1:
; GFX11W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX11W32-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX11W32-NEXT: v_mov_b32_e32 v2, 0
; GFX11W32-NEXT: s_waitcnt lgkmcnt(0)
; GFX11W32-NEXT: s_mul_i32 s2, s0, s2
; GFX11W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11W32-NEXT: v_mov_b32_e32 v1, s2
+; GFX11W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX11W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], 0 idxen glc
; GFX11W32-NEXT: .LBB6_2:
; GFX11W32-NEXT: s_or_b32 exec_lo, exec_lo, s1
@@ -2026,30 +2016,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12W64-LABEL: sub_i32_uniform:
; GFX12W64: ; %bb.0: ; %entry
-; GFX12W64-NEXT: s_load_b32 s6, s[4:5], 0x44
-; GFX12W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX12W64-NEXT: s_load_b32 s2, s[4:5], 0x44
+; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX12W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W64-NEXT: ; implicit-def: $vgpr1
-; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX12W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX12W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX12W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX12W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W64-NEXT: s_cbranch_execz .LBB6_2
; GFX12W64-NEXT: ; %bb.1:
; GFX12W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX12W64-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX12W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX12W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX12W64-NEXT: v_mov_b32_e32 v1, s3
; GFX12W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W64-NEXT: .LBB6_2:
; GFX12W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX12W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12W64-NEXT: s_wait_kmcnt 0x0
-; GFX12W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX12W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX12W64-NEXT: s_wait_loadcnt 0x0
; GFX12W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX12W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2062,20 +2051,19 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX12W32-LABEL: sub_i32_uniform:
; GFX12W32: ; %bb.0: ; %entry
; GFX12W32-NEXT: s_load_b32 s0, s[4:5], 0x44
-; GFX12W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX12W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12W32-NEXT: ; implicit-def: $vgpr1
-; GFX12W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX12W32-NEXT: s_mov_b32 s1, exec_lo
; GFX12W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12W32-NEXT: s_cbranch_execz .LBB6_2
; GFX12W32-NEXT: ; %bb.1:
; GFX12W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
-; GFX12W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12W32-NEXT: s_wait_kmcnt 0x0
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12W32-NEXT: s_mul_i32 s2, s0, s2
-; GFX12W32-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12W32-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
; GFX12W32-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX12W32-NEXT: .LBB6_2:
@@ -2094,29 +2082,29 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13W64-LABEL: sub_i32_uniform:
; GFX13W64: ; %bb.0: ; %entry
-; GFX13W64-NEXT: s_load_b32 s6, s[4:5], 0x44 nv
-; GFX13W64-NEXT: s_mov_b64 s[2:3], exec
+; GFX13W64-NEXT: s_load_b32 s2, s[4:5], 0x44 nv
+; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
-; GFX13W64-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W64-NEXT: ; implicit-def: $vgpr1
-; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W64-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
+; GFX13W64-NEXT: s_mov_b64 s[0:1], exec
+; GFX13W64-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX13W64-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX13W64-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W64-NEXT: s_cbranch_execz .LBB6_2
; GFX13W64-NEXT: ; %bb.1:
; GFX13W64-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W64-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: s_mul_i32 s2, s6, s2
-; GFX13W64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX13W64-NEXT: v_mov_b32_e32 v1, s2
+; GFX13W64-NEXT: s_mul_i32 s3, s2, s3
+; GFX13W64-NEXT: v_mov_b32_e32 v2, 0
+; GFX13W64-NEXT: v_mov_b32_e32 v1, s3
; GFX13W64-NEXT: buffer_atomic_sub_u32 v1, v2, s[8:11], null idxen th:TH_ATOMIC_RETURN
; GFX13W64-NEXT: .LBB6_2:
; GFX13W64-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX13W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX13W64-NEXT: s_wait_kmcnt 0x0
-; GFX13W64-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX13W64-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX13W64-NEXT: s_wait_loadcnt 0x0
; GFX13W64-NEXT: v_readfirstlane_b32 s2, v1
; GFX13W64-NEXT: v_mov_b32_e32 v1, 0
@@ -2128,16 +2116,16 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX13W32-LABEL: sub_i32_uniform:
; GFX13W32: ; %bb.0: ; %entry
; GFX13W32-NEXT: s_load_b32 s0, s[4:5], 0x44 nv
-; GFX13W32-NEXT: s_mov_b32 s2, exec_lo
+; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
-; GFX13W32-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX13W32-NEXT: ; implicit-def: $vgpr1
-; GFX13W32-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s1
+; GFX13W32-NEXT: s_mov_b32 s1, exec_lo
; GFX13W32-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX13W32-NEXT: s_cbranch_execz .LBB6_2
; GFX13W32-NEXT: ; %bb.1:
; GFX13W32-NEXT: s_load_b128 s[8:11], s[4:5], 0x34 nv
-; GFX13W32-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX13W32-NEXT: s_wait_kmcnt 0x0
; GFX13W32-NEXT: s_mul_i32 s2, s0, s2
; GFX13W32-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
index ca7cc989c510c..fd91f43ac3868 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
@@ -11,7 +11,7 @@
; GFX90A-HW: Hardware instruction generated for atomic fadd operation at memory scope singlethread-one-as due to an unsafe request.
; GFX90A-HW-LABEL: atomic_add_unsafe_hw:
-; GFX90A-HW: ds_add_f64 v2, v[0:1]
+; GFX90A-HW: ds_add_f64 v0, v[2:3]
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw(ptr addrspace(3) %ptr) #0 {
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
index 9ac62fadce6c6..b29d3cfe46f96 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
@@ -58,41 +58,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX908-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX908-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 killed [[COPY6]], [[COPY7]], implicit $exec
; GFX908-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 killed [[COPY5]], killed [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX908-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX908-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX908-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, killed [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX908-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX908-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 killed [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]]
- ; GFX908-NEXT: early-clobber %1:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
+ ; GFX908-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX908-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX908-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX908-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: S_BRANCH %bb.2
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.2 (%ir-block.31):
+ ; GFX908-NEXT: bb.2 (%ir-block.17):
; GFX908-NEXT: successors: %bb.3(0x80000000)
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX908-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY %1
- ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX908-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX908-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: bb.3.Flow:
; GFX908-NEXT: successors: %bb.4(0x80000000)
; GFX908-NEXT: {{ $}}
; GFX908-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: {{ $}}
- ; GFX908-NEXT: bb.4 (%ir-block.33):
+ ; GFX908-NEXT: bb.4 (%ir-block.19):
; GFX908-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX908-NEXT: S_ENDPGM 0
;
@@ -120,41 +103,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX90A_GFX942-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX90A_GFX942-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 killed [[COPY6]], [[COPY7]], implicit $exec
; GFX90A_GFX942-NEXT: [[V_MBCNT_HI_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_HI_U32_B32_e64 killed [[COPY5]], killed [[V_MBCNT_LO_U32_B32_e64_]], implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX90A_GFX942-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
- ; GFX90A_GFX942-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 273, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 274, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 276, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 280, 15, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp4:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_3]], 322, 10, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_MOV_B32_dpp4]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_dpp5:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_4]], 323, 12, 15, 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[V_ADD_F32_e64_5:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_4]], 0, killed [[V_MOV_B32_dpp5]], 0, 0, implicit $mode, implicit $exec
- ; GFX90A_GFX942-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 63
- ; GFX90A_GFX942-NEXT: [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 killed [[V_ADD_F32_e64_5]], killed [[S_MOV_B32_1]]
- ; GFX90A_GFX942-NEXT: early-clobber %1:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
+ ; GFX90A_GFX942-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX90A_GFX942-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX90A_GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX90A_GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: S_BRANCH %bb.2
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: bb.2 (%ir-block.31):
+ ; GFX90A_GFX942-NEXT: bb.2 (%ir-block.17):
; GFX90A_GFX942-NEXT: successors: %bb.3(0x80000000)
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX90A_GFX942-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY %1
- ; GFX90A_GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX90A_GFX942-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX90A_GFX942-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY8]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX90A_GFX942-NEXT: {{ $}}
; GFX90A_GFX942-NEXT: bb.3.Flow:
; GFX90A_GFX942-NEXT: successors: %bb.4(0x80000000)
; GFX90A_GFX942-NEXT: {{ $}}
; GFX90A_GFX942-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: {{ $}}
- ; GFX90A_GFX942-NEXT: bb.4 (%ir-block.33):
+ ; GFX90A_GFX942-NEXT: bb.4 (%ir-block.19):
; GFX90A_GFX942-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A_GFX942-NEXT: S_ENDPGM 0
;
@@ -178,38 +144,24 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
; GFX11_GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11_GFX12-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
- ; GFX11_GFX12-NEXT: [[DEF:%[0-9]+]]:sreg_32_xm0_xexec = IMPLICIT_DEF
- ; GFX11_GFX12-NEXT: [[V_SET_INACTIVE_B32_:%[0-9]+]]:vgpr_32 = V_SET_INACTIVE_B32 0, [[COPY]], 0, [[V_MOV_B32_e32_]], killed [[DEF]], implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_SET_INACTIVE_B32_]], 353, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_SET_INACTIVE_B32_]], 0, killed [[V_MOV_B32_dpp]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp1:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_]], 354, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_]], 0, killed [[V_MOV_B32_dpp1]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp2:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_1]], 356, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_1]], 0, killed [[V_MOV_B32_dpp2]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_dpp3:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_ADD_F32_e64_2]], 360, 15, 15, 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_3:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_2]], 0, killed [[V_MOV_B32_dpp3]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
- ; GFX11_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF1]]
- ; GFX11_GFX12-NEXT: [[V_PERMLANEX16_B32_e64_:%[0-9]+]]:vgpr_32 = V_PERMLANEX16_B32_e64 0, [[V_ADD_F32_e64_3]], 0, [[S_MOV_B32_]], 0, [[S_MOV_B32_]], [[COPY5]], 0, implicit $exec
- ; GFX11_GFX12-NEXT: [[V_ADD_F32_e64_4:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[V_ADD_F32_e64_3]], 0, killed [[V_PERMLANEX16_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; GFX11_GFX12-NEXT: early-clobber %1:vgpr_32 = STRICT_WWM killed [[V_ADD_F32_e64_4]], implicit $exec
+ ; GFX11_GFX12-NEXT: [[WAVE_REDUCE_FADD_PSEUDO_F32_:%[0-9]+]]:sgpr_32 = WAVE_REDUCE_FADD_PSEUDO_F32 [[COPY]], 2, implicit $exec
+ ; GFX11_GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[WAVE_REDUCE_FADD_PSEUDO_F32_]]
; GFX11_GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_LO_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX11_GFX12-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_EQ_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: S_BRANCH %bb.2
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: bb.2 (%ir-block.24):
+ ; GFX11_GFX12-NEXT: bb.2 (%ir-block.13):
; GFX11_GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11_GFX12-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_1]], %1, [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
+ ; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11_GFX12-NEXT: GLOBAL_ATOMIC_ADD_F32_SADDR killed [[V_MOV_B32_e32_]], [[COPY5]], [[COPY3]], 0, 0, implicit $exec :: (load store syncscope("wavefront") monotonic (s32) on %ir.ptr, addrspace 1)
; GFX11_GFX12-NEXT: {{ $}}
; GFX11_GFX12-NEXT: bb.3.Flow:
; GFX11_GFX12-NEXT: successors: %bb.4(0x80000000)
; GFX11_GFX12-NEXT: {{ $}}
; GFX11_GFX12-NEXT: SI_END_CF [[SI_IF1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: bb.4 (%ir-block.26):
+ ; GFX11_GFX12-NEXT: bb.4 (%ir-block.15):
; GFX11_GFX12-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11_GFX12-NEXT: S_ENDPGM 0
%ret = atomicrmw fadd ptr addrspace(1) %ptr, float %data syncscope("wavefront") monotonic, !amdgpu.no.fine.grained.memory !0, !atomic.ignore.denormal.mode !0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
index 295183b046bf0..2c6183f47a91f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-scan.ll
@@ -11,15 +11,13 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -38,15 +36,13 @@ define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -65,15 +61,13 @@ define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -92,15 +86,13 @@ define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -120,15 +112,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -154,15 +144,13 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -183,15 +171,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -216,15 +202,13 @@ define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -241,15 +225,13 @@ define amdgpu_kernel void @atomic_add_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -273,15 +255,13 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -300,15 +280,13 @@ define amdgpu_kernel void @atomic_add_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -333,12 +311,13 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -357,15 +336,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -388,12 +368,13 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -414,15 +395,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -444,12 +426,13 @@ define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -466,15 +449,16 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -495,12 +479,13 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -519,15 +504,16 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.and.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -1, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = and i32 [[TMP11]], [[TMP12]]
; IR-NEXT: store i32 [[TMP13]], ptr addrspace(1) [[OUT2:%.*]], align 4
; IR-NEXT: ret void
@@ -549,15 +535,13 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -576,15 +560,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -610,15 +592,13 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -639,15 +619,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[GEP]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -672,15 +650,13 @@ define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -697,15 +673,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[OUT:%.*]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -729,15 +703,13 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -756,15 +728,13 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 [[IN:%.*]], [[TMP7]]
+; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[IN:%.*]], i32 1)
; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
+; IR: 8:
; IR-NEXT: [[TMP11:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[TMP8]] seq_cst, align 4
; IR-NEXT: br label [[TMP12]]
-; IR: 12:
+; IR: 10:
; IR-NEXT: [[TMP13:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP11]], [[TMP10]] ]
; IR-NEXT: [[TMP14:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP13]])
; IR-NEXT: [[TMP15:%.*]] = mul i32 [[IN]], [[TMP5]]
@@ -789,12 +759,13 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN]] seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -813,15 +784,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -845,12 +817,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -871,15 +844,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -902,12 +876,13 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -924,15 +899,16 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -954,12 +930,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -978,15 +955,16 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.max.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 -2147483648, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1009,12 +987,13 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1033,15 +1012,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1065,12 +1045,13 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1091,15 +1072,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1122,12 +1104,13 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1144,15 +1127,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrsp
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1174,12 +1158,13 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1198,15 +1183,16 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.umax.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 0, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp ugt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1229,12 +1215,13 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in)
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1253,15 +1240,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1285,12 +1273,13 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1311,15 +1300,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[GEP]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1342,12 +1332,13 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1364,15 +1355,16 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspa
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[OUT:%.*]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[OUT:%.*]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
@@ -1394,12 +1386,13 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in,
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: ret void
;
entry:
@@ -1418,15 +1411,16 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[IN:%.*]] = call i32 @llvm.amdgcn.wave.reduce.min.i32(i32 [[IN1:%.*]], i32 1)
; IR-NEXT: [[TMP6:%.*]] = icmp eq i32 [[TMP5]], 0
; IR-NEXT: br i1 [[TMP6]], label [[TMP7:%.*]], label [[TMP9:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[IN:%.*]] syncscope("workgroup") seq_cst, align 4
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[IN]] syncscope("workgroup") seq_cst, align 4
; IR-NEXT: br label [[TMP9]]
-; IR: 9:
+; IR: 10:
; IR-NEXT: [[TMP10:%.*]] = phi i32 [ poison, [[ENTRY:%.*]] ], [ [[TMP8]], [[TMP7]] ]
; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP10]])
-; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN]]
+; IR-NEXT: [[TMP12:%.*]] = select i1 [[TMP6]], i32 2147483647, i32 [[IN1]]
; IR-NEXT: [[TMP13:%.*]] = icmp slt i32 [[TMP11]], [[TMP12]]
; IR-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
; IR-NEXT: store i32 [[TMP14]], ptr addrspace(1) [[OUT2:%.*]], align 4
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
index 70880eb63b620..f079df2591b83 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd-wrong-subtarget.ll
@@ -5,30 +5,32 @@ define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrsp
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_readfirstlane_b32 s8, v1
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_4
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GCN-NEXT: s_mov_b64 s[6:7], 0
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_load_dword s8, s[4:5], 0x0
+; GCN-NEXT: s_load_dword s0, s[4:5], 0x0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s8
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: .LBB0_2: ; %atomicrmw.start
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_madmk_f32 v4, v2, 0x40800000, v5
-; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[4:5] glc
+; GCN-NEXT: v_mov_b32_e32 v4, v1
+; GCN-NEXT: v_add_f32_e32 v3, s8, v4
+; GCN-NEXT: global_atomic_cmpswap v1, v2, v[3:4], s[4:5] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_wbinvl1
-; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v5
+; GCN-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GCN-NEXT: s_or_b64 s[6:7], s[0:1], s[6:7]
; GCN-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GCN-NEXT: s_cbranch_execnz .LBB0_2
@@ -52,17 +54,19 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addr
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll b/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
index 3c93f27681f75..478351c82f6aa 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomics-fp-wrong-subtarget.ll
@@ -12,17 +12,19 @@ define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addr
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[0:1], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GCN-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GCN-NEXT: v_readfirstlane_b32 s0, v0
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, s0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
index a09643b859232..376653c5b3d0c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
@@ -7,36 +7,61 @@
; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers
diff erent scopes.
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float
-; IR-NEXT: [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]])
-; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-NEXT: [[TMP20:%.*]] = fmul float [[VAL]], [[TMP19]]
-; IR-NEXT: [[TMP21:%.*]] = fadd float [[TMP18]], [[TMP20]]
-; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
-; IR-NEXT: br label [[TMP23]]
-; IR: 23:
-; IR-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
-; IR-NEXT: ret float [[TMP24]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = fmul float [[VAL]], [[TMP16]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = fadd float [[TMP15]], [[TMP17]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret float [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-DPP-NEXT: [[TMP17:%.*]] = fmul float [[VAL]], [[TMP16]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = fadd float [[TMP15]], [[TMP17]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret float [[TMP21]]
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
ret float %result
@@ -133,64 +158,58 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_scope_agent_sco
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -200,24 +219,24 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_one_as_scope_un
define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -227,11 +246,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -242,31 +261,31 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -274,9 +293,9 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -289,64 +308,58 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_one_as_scope_un
define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -356,24 +369,24 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_uni_value_agent_scope_str
define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -383,11 +396,11 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -398,31 +411,31 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -430,9 +443,9 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fsub.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -444,32 +457,61 @@ define amdgpu_ps float @global_atomic_fsub_uni_address_div_value_agent_scope_str
}
define amdgpu_ps float @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-NEXT: [[TMP17:%.*]] = call float @llvm.minnum.f32(float [[TMP14]], float [[TMP16]])
-; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
-; IR-NEXT: br label [[TMP19]]
-; IR: 19:
-; IR-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
-; IR-NEXT: ret float [[TMP20]]
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], float +qnan, float [[VAL]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP17]])
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret float [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to float
+; IR-DPP-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], float +qnan, float [[VAL]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP17]])
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], float [[TMP15]], float [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret float [[TMP21]]
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
ret float %result
@@ -566,56 +608,58 @@ define amdgpu_ps float @global_atomic_fmin_uni_address_div_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL1]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
; IR-ITERATIVE-NEXT: br label [[TMP19]]
-; IR-ITERATIVE: 19:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP20]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP13:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], float +qnan, float [[VAL1]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], float [[TMP14]], float [[TMP17]]
; IR-DPP-NEXT: br label [[TMP19]]
-; IR-DPP: 19:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP20:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-DPP-NEXT: ret float [[TMP20]]
;
@@ -625,24 +669,24 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_uni_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -652,11 +696,11 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -667,31 +711,31 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -699,9 +743,9 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP30]], float [[TMP31]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP30]], float [[TMP31]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -714,64 +758,58 @@ define amdgpu_ps float @global_atomic_fmax_uni_address_div_value_agent_scope_uns
define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret float [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL]], float [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP18]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], float [[TMP18]], float [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi float [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret float [[TMP24]]
;
@@ -781,24 +819,24 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_uni_value_system_scope_st
define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23:%.*]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi float [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP14]], float [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], float [[TMP14]], float [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -808,11 +846,11 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call float @llvm.amdgcn.writelane.f32(float [[ACCUMULATOR]], i32 [[TMP20]], float [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -823,31 +861,31 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -855,9 +893,9 @@ define amdgpu_ps float @global_atomic_fadd_uni_address_div_value_system_scope_st
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi float [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP30]], float [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], float [[TMP30]], float [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -977,36 +1015,61 @@ define amdgpu_ps float @global_atomic_fadd_div_address_div_value_system_scope_st
}
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double
-; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]])
-; IR-NEXT: [[TMP19:%.*]] = uitofp i32 [[TMP8]] to double
-; IR-NEXT: [[TMP20:%.*]] = fmul double [[VAL]], [[TMP19]]
-; IR-NEXT: [[TMP21:%.*]] = fadd double [[TMP18]], [[TMP20]]
-; IR-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
-; IR-NEXT: br label [[TMP23]]
-; IR: 23:
-; IR-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
-; IR-NEXT: ret double [[TMP24]]
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = fmul double [[VAL]], [[TMP16]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = fadd double [[TMP15]], [[TMP17]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret double [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-DPP-NEXT: [[TMP17:%.*]] = fmul double [[VAL]], [[TMP16]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = fadd double [[TMP15]], [[TMP17]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret double [[TMP21]]
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
ret double %result
@@ -1103,64 +1166,58 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_scope_a
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1170,24 +1227,24 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_one_as_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1197,11 +1254,11 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1212,31 +1269,31 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1244,9 +1301,9 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1259,64 +1316,58 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_one_as_
define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1326,24 +1377,24 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_uni_value_agent_s
define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1353,11 +1404,11 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1368,31 +1419,31 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1400,9 +1451,9 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fsub.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1414,32 +1465,61 @@ define amdgpu_ps double @global_atomic_fsub_double_uni_address_div_value_agent_s
}
define amdgpu_ps double @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]])
-; IR-NEXT: [[TMP15:%.*]] = uitofp i32 [[TMP8]] to double
-; IR-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-NEXT: [[TMP17:%.*]] = call double @llvm.minnum.f64(double [[TMP14]], double [[TMP16]])
-; IR-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
-; IR-NEXT: br label [[TMP19]]
-; IR: 19:
-; IR-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
-; IR-NEXT: ret double [[TMP20]]
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], double +qnan, double [[VAL]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP17]])
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-ITERATIVE-NEXT: br label [[TMP20]]
+; IR-ITERATIVE: 20:
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-ITERATIVE-NEXT: ret double [[TMP21]]
+;
+; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP20:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: [[TMP14:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP12]], [[TMP11]] ]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP14]])
+; IR-DPP-NEXT: [[TMP16:%.*]] = uitofp i32 [[TMP8]] to double
+; IR-DPP-NEXT: [[TMP17:%.*]] = select i1 [[TMP10]], double +qnan, double [[VAL]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP17]])
+; IR-DPP-NEXT: [[TMP19:%.*]] = select i1 [[TMP10]], double [[TMP15]], double [[TMP18]]
+; IR-DPP-NEXT: br label [[TMP20]]
+; IR-DPP: 20:
+; IR-DPP-NEXT: [[TMP21:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP19]], [[TMP13]] ]
+; IR-DPP-NEXT: ret double [[TMP21]]
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
ret double %result
@@ -1536,56 +1616,58 @@ define amdgpu_ps double @global_atomic_fmin_double_uni_address_div_value_agent_s
define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL1]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
; IR-ITERATIVE-NEXT: br label [[TMP19]]
-; IR-ITERATIVE: 19:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP20]]
;
; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP19:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP13:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP11]], [[TMP10]] ]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = select i1 [[TMP9]], double +qnan, double [[VAL1]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP18:%.*]] = select i1 [[TMP9]], double [[TMP14]], double [[TMP17]]
; IR-DPP-NEXT: br label [[TMP19]]
-; IR-DPP: 19:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP20:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP18]], [[TMP12]] ]
; IR-DPP-NEXT: ret double [[TMP20]]
;
@@ -1595,24 +1677,24 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_uni_value_agent_
define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1622,11 +1704,11 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1637,31 +1719,31 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic__fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1669,9 +1751,9 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP30]], double [[TMP31]], metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP30]], double [[TMP31]], metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
@@ -1684,64 +1766,58 @@ define amdgpu_ps double @global_atomic__fmax_double_uni_address_div_value_agent_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-ITERATIVE-NEXT: br label [[TMP23]]
-; IR-ITERATIVE: 23:
+; IR-ITERATIVE: 20:
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-ITERATIVE-NEXT: ret double [[TMP24]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP23:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: [[TMP17:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP15]], [[TMP14]] ]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP17]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP8]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL]], double [[TMP19]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP18]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP22:%.*]] = select i1 [[TMP13]], double [[TMP18]], double [[TMP21]]
; IR-DPP-NEXT: br label [[TMP23]]
-; IR-DPP: 23:
+; IR-DPP: 20:
; IR-DPP-NEXT: [[TMP24:%.*]] = phi double [ poison, [[TMP0:%.*]] ], [ [[TMP22]], [[TMP16]] ]
; IR-DPP-NEXT: ret double [[TMP24]]
;
@@ -1751,24 +1827,24 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_uni_value_system_
define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
; IR-ITERATIVE: 10:
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23:%.*]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
; IR-ITERATIVE: 12:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = phi double [ poison, [[COMPUTEEND:%.*]] ], [ [[TMP11]], [[TMP10:%.*]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP13]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP14]], double [[TMP22:%.*]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = select i1 [[TMP28:%.*]], double [[TMP14]], double [[TMP15]]
; IR-ITERATIVE-NEXT: br label [[TMP17]]
; IR-ITERATIVE: 17:
@@ -1778,11 +1854,11 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP23]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[OLDVALUEPHI:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP22]], [[COMPUTELOOP]] ]
; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP26:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP20:%.*]] = trunc i64 [[TMP19]] to i32
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP20]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP22]] = call double @llvm.amdgcn.writelane.f64(double [[ACCUMULATOR]], i32 [[TMP20]], double [[OLDVALUEPHI]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP23]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP21]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP24:%.*]] = shl i64 1, [[TMP19]]
; IR-ITERATIVE-NEXT: [[TMP25:%.*]] = xor i64 [[TMP24]], -1
; IR-ITERATIVE-NEXT: [[TMP26]] = and i64 [[ACTIVEBITS]], [[TMP25]]
@@ -1793,31 +1869,31 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-ITERATIVE-NEXT: br i1 [[TMP28]], label [[TMP10]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP34:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP21]], i32 312, i32 15, i32 15, i1 false) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP24:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP23]]) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP25:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP25]], label [[TMP26:%.*]], label [[TMP28:%.*]]
; IR-DPP: 26:
@@ -1825,9 +1901,9 @@ define amdgpu_ps double @global_atomic_fadd_double_uni_address_div_value_system_
; IR-DPP-NEXT: br label [[TMP28]]
; IR-DPP: 28:
; IR-DPP-NEXT: [[TMP29:%.*]] = phi double [ poison, [[TMP2]] ], [ [[TMP27]], [[TMP26]] ]
-; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP30:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[TMP29]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP31:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP32:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP30]], double [[TMP31]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR2]]
; IR-DPP-NEXT: [[TMP33:%.*]] = select i1 [[TMP25]], double [[TMP30]], double [[TMP32]]
; IR-DPP-NEXT: br label [[TMP34]]
; IR-DPP: 34:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
index 82d478b5f64b7..e0b0d8a4825e8 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
@@ -22,15 +22,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1), ptr addrspace(1) %val
; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-NEXT: [[TMP9:%.*]] = mul i32 [[VALUE]], [[TMP8]]
+; IR-NEXT: [[TMP9:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[VALUE]], i32 1)
; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
-; IR: 11:
+; IR: 9:
; IR-NEXT: [[TMP12:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP9]] seq_cst, align 4
; IR-NEXT: br label [[TMP13]]
-; IR: 13:
+; IR: 11:
; IR-NEXT: ret void
;
entry:
@@ -71,28 +69,14 @@ define amdgpu_kernel void @divergent_value(ptr addrspace(1) %out, ptr addrspace(
; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
-; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR: 7:
-; IR-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP13:%.*]] seq_cst, align 4
-; IR-NEXT: br label [[TMP9:%.*]]
-; IR: 9:
+; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.wave.reduce.add.i32(i32 [[VALUE]], i32 1)
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label [[TMP10:%.*]], label [[TMP9:%.*]]
+; IR: 8:
+; IR-NEXT: [[TMP8:%.*]] = atomicrmw add ptr addrspace(1) [[GEP]], i32 [[TMP13]] seq_cst, align 4
+; IR-NEXT: br label [[TMP9]]
+; IR: 10:
; IR-NEXT: ret void
-; IR: ComputeLoop:
-; IR-NEXT: [[ACCUMULATOR:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP13]], [[COMPUTELOOP]] ]
-; IR-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP6]], [[ENTRY]] ], [ [[TMP16:%.*]], [[COMPUTELOOP]] ]
-; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
-; IR-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VALUE]], i32 [[TMP11]])
-; IR-NEXT: [[TMP13]] = add i32 [[ACCUMULATOR]], [[TMP12]]
-; IR-NEXT: [[TMP14:%.*]] = shl i64 1, [[TMP10]]
-; IR-NEXT: [[TMP15:%.*]] = xor i64 [[TMP14]], -1
-; IR-NEXT: [[TMP16]] = and i64 [[ACTIVEBITS]], [[TMP15]]
-; IR-NEXT: [[TMP17:%.*]] = icmp eq i64 [[TMP16]], 0
-; IR-NEXT: br i1 [[TMP17]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR: ComputeEnd:
-; IR-NEXT: [[TMP18:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP18]], label [[TMP7:%.*]], label [[TMP9]]
;
entry:
%divergent_value.kernarg.segment = call nonnull align 16 dereferenceable(52) ptr addrspace(4) @llvm.amdgcn.kernarg.segment.ptr()
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
index 1aa0da2f696ac..402a468c2599c 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
@@ -11,16 +11,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 1)
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 9:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-ITERATIVE-NEXT: br label [[TMP14]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_value(
@@ -30,16 +27,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-DPP-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-DPP-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-DPP-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 2)
; IR-DPP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-DPP: 12:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP14]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
@@ -57,28 +51,14 @@ define amdgpu_kernel void @global_atomic_fadd_div_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 8:
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP14:%.*]] seq_cst, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP10:%.*]]
-; IR-ITERATIVE: 10:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP6]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP8]], label [[TMP11:%.*]], label [[TMP10:%.*]]
+; IR-ITERATIVE: 9:
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP14]] seq_cst, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP10]]
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP0:%.*]] ], [ [[TMP14]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP7]], [[TMP0]] ], [ [[TMP17:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[DIVVALUE]], i32 [[TMP12]])
-; IR-ITERATIVE-NEXT: [[TMP14]] = fadd float [[ACCUMULATOR]], [[TMP13]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = shl i64 1, [[TMP11]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = xor i64 [[TMP15]], -1
-; IR-ITERATIVE-NEXT: [[TMP17]] = and i64 [[ACTIVEBITS]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP18]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP6]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP19]], label [[TMP8:%.*]], label [[TMP10]]
;
; IR-DPP-LABEL: @global_atomic_fadd_div_value(
; IR-DPP-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -89,27 +69,13 @@ define amdgpu_kernel void @global_atomic_fadd_div_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[DIVVALUE]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP8:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP7]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP19]], i32 63)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP20]])
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 2)
; IR-DPP-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]]
-; IR-DPP: 23:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP24:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP21]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP25]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
@@ -126,16 +92,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 1)
; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-ITERATIVE: 12:
+; IR-ITERATIVE: 9:
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-ITERATIVE-NEXT: br label [[TMP14]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_value(
@@ -145,16 +108,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP1]])
-; IR-DPP-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; IR-DPP-NEXT: [[TMP9:%.*]] = uitofp i32 [[TMP8]] to float
-; IR-DPP-NEXT: [[TMP10:%.*]] = fmul float 4.000000e+00, [[TMP9]]
+; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float 4.000000e+00, i32 2)
; IR-DPP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP11]], label [[TMP12:%.*]], label [[TMP14:%.*]]
-; IR-DPP: 12:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP13:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP10]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP14]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
@@ -172,28 +132,14 @@ define amdgpu_kernel void @global_atomic_fsub_div_value(ptr addrspace(1) %ptr) {
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 8:
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP14:%.*]] seq_cst, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP10:%.*]]
-; IR-ITERATIVE: 10:
+; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP6]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP8]], label [[TMP11:%.*]], label [[TMP10:%.*]]
+; IR-ITERATIVE: 9:
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP14]] seq_cst, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP10]]
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP0:%.*]] ], [ [[TMP14]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP7]], [[TMP0]] ], [ [[TMP17:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i32
-; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[DIVVALUE]], i32 [[TMP12]])
-; IR-ITERATIVE-NEXT: [[TMP14]] = fadd float [[ACCUMULATOR]], [[TMP13]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = shl i64 1, [[TMP11]]
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = xor i64 [[TMP15]], -1
-; IR-ITERATIVE-NEXT: [[TMP17]] = and i64 [[ACTIVEBITS]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP18]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = icmp eq i32 [[TMP6]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP19]], label [[TMP8:%.*]], label [[TMP10]]
;
; IR-DPP-LABEL: @global_atomic_fsub_div_value(
; IR-DPP-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
@@ -204,27 +150,13 @@ define amdgpu_kernel void @global_atomic_fsub_div_value(ptr addrspace(1) %ptr) {
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP2]], i32 0)
; IR-DPP-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP4]], i32 [[TMP5]])
-; IR-DPP-NEXT: [[TMP7:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[DIVVALUE]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP8:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP7]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP19]], i32 63)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP20]])
+; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[DIVVALUE]], i32 2)
; IR-DPP-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP6]], 0
; IR-DPP-NEXT: br i1 [[TMP22]], label [[TMP23:%.*]], label [[TMP25:%.*]]
-; IR-DPP: 23:
+; IR-DPP: 9:
; IR-DPP-NEXT: [[TMP24:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP21]] seq_cst, align 4
; IR-DPP-NEXT: br label [[TMP25]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: ret void
;
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
index bda6511857b11..5cf8adc778061 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
@@ -7,29 +7,47 @@
; strategies are valid for only divergent values. This optimization is valid for divergent addresses. Test also covers
diff erent scopes.
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to float
-; IR-NEXT: [[TMP12:%.*]] = fmul float [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: br label [[TMP17]]
-; IR: 17:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
ret void
@@ -46,30 +64,16 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = fadd float [[ACCUMULATOR]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_scope_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -81,29 +85,15 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd float [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd float [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd float [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd float [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = fadd float [[TMP19]], [[TMP20]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic, align 4
@@ -112,51 +102,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_scope_agent_scop
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7:[0-9]+]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8:[0-9]+]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2:[0-9]+]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
@@ -165,73 +149,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_one_as_scope_uns
define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("one-as") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("one-as") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("one-as") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
@@ -240,51 +196,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_one_as_scope_uns
define amdgpu_ps void @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -293,73 +243,45 @@ define amdgpu_ps void @global_atomic_fsub_uni_address_uni_value_agent_scope_stri
define amdgpu_ps void @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fsub ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -367,25 +289,47 @@ define amdgpu_ps void @global_atomic_fsub_uni_address_div_value_agent_scope_stri
}
define amdgpu_ps void @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, float inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: br label [[TMP13]]
-; IR: 13:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
ret void
@@ -402,30 +346,16 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.minnum.f32(float [[ACCUMULATOR]], float [[TMP16]])
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmin_uni_address_div_value_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -437,29 +367,15 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.minnum.f32(float [[TMP9]], float [[TMP10]])
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.minnum.f32(float [[TMP11]], float [[TMP12]])
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.minnum.f32(float [[TMP13]], float [[TMP14]])
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.minnum.f32(float [[TMP15]], float [[TMP16]])
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.minnum.f32(float [[TMP17]], float [[TMP18]])
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.minnum.f32(float [[TMP19]], float [[TMP20]])
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fmin.f32(float [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -468,43 +384,45 @@ define amdgpu_ps void @global_atomic_fmin_uni_address_div_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL:%.*]] syncscope("agent") monotonic, align 4
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[VAL]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
-; IR-DPP-NEXT: br label [[TMP13]]
; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -513,73 +431,45 @@ define amdgpu_ps void @global_atomic_fmax_uni_address_uni_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, float %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.maxnum.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP9]], float [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP11]], float [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP13]], float [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP15]], float [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP17]], float [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float +qnan, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.maxnum.f32(float [[TMP19]], float [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fmax.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("agent") monotonic
@@ -588,51 +478,45 @@ define amdgpu_ps void @global_atomic_fmax_uni_address_div_value_agent_scope_unsa
define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.uitofp.f32.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.experimental.constrained.fmul.f32(float [[VAL:%.*]], float [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4
@@ -641,73 +525,45 @@ define amdgpu_ps void @global_atomic_fadd_uni_address_uni_value_system_scope_str
define amdgpu_ps void @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, float %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17:%.*]] monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP17]] monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi float [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call float @llvm.experimental.constrained.fadd.f32(float [[ACCUMULATOR]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call float @llvm.amdgcn.set.inactive.f32(float [[VAL:%.*]], float -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP9]], float [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP11]], float [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP13]], float [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP15]], float [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP17]], float [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float -0.000000e+00, float [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float [[TMP19]], float [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.strict.wwm.f32(float [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call float @llvm.amdgcn.wave.reduce.fadd.f32(float [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], float [[TMP23]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic, align 4
@@ -823,29 +679,47 @@ define amdgpu_ps void @global_atomic_fadd_div_address_div_value_system_scope_str
}
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]])
-; IR-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-NEXT: [[TMP11:%.*]] = uitofp i32 [[TMP10]] to double
-; IR-NEXT: [[TMP12:%.*]] = fmul double [[VAL:%.*]], [[TMP11]]
-; IR-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR: 14:
-; IR-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 4
-; IR-NEXT: br label [[TMP16]]
-; IR: 16:
-; IR-NEXT: br label [[TMP17]]
-; IR: 17:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 4
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
ret void
@@ -862,30 +736,16 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = fadd double [[ACCUMULATOR]], [[TMP16]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_scope_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -897,29 +757,15 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = fadd double [[TMP9]], [[TMP10]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = fadd double [[TMP11]], [[TMP12]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = fadd double [[TMP13]], [[TMP14]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = fadd double [[TMP15]], [[TMP16]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = fadd double [[TMP17]], [[TMP18]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = fadd double [[TMP19]], [[TMP20]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic, align 4
@@ -928,51 +774,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_scope_age
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
@@ -981,73 +821,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_one_as_sc
define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("one-as") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("one-as") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_one_as_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("one-as") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
@@ -1056,51 +868,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_one_as_sc
define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1109,73 +915,45 @@ define amdgpu_ps void @global_atomic_fsub_double_uni_address_uni_value_agent_sco
define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fsub_double_uni_address_div_value_agent_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fsub ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fsub ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1183,25 +961,47 @@ define amdgpu_ps void @global_atomic_fsub_double_uni_address_div_value_agent_sco
}
define amdgpu_ps void @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) inreg %ptr, double inreg %val) #0 {
-; IR-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
-; IR-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
-; IR-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
-; IR: 2:
-; IR-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
-; IR-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
-; IR-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
-; IR-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR: 10:
-; IR-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
-; IR-NEXT: br label [[TMP12]]
-; IR: 12:
-; IR-NEXT: br label [[TMP13]]
-; IR: 13:
-; IR-NEXT: ret void
+; IR-ITERATIVE-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-ITERATIVE: 2:
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP14]]
+; IR-ITERATIVE: 14:
+; IR-ITERATIVE-NEXT: ret void
+;
+; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe(
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
+; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP14:%.*]]
+; IR-DPP: 2:
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
+; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
+; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
+; IR-DPP-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-DPP-NEXT: br i1 [[TMP10]], label [[TMP11:%.*]], label [[TMP13:%.*]]
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP12:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP9]] syncscope("agent") monotonic, align 8
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP14]]
+; IR-DPP: 14:
+; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
ret void
@@ -1218,30 +1018,16 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 1)
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true)
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]])
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.minnum.f64(double [[ACCUMULATOR]], double [[TMP16]])
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmin_double_uni_address_div_value_agent_scope_unsafe(
; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live()
@@ -1253,29 +1039,15 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0)
; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]])
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan)
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.minnum.f64(double [[TMP9]], double [[TMP10]])
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.minnum.f64(double [[TMP11]], double [[TMP12]])
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.minnum.f64(double [[TMP13]], double [[TMP14]])
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.minnum.f64(double [[TMP15]], double [[TMP16]])
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.minnum.f64(double [[TMP17]], double [[TMP18]])
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false)
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.minnum.f64(double [[TMP19]], double [[TMP20]])
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63)
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]])
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fmin.f64(double [[VAL:%.*]], i32 2)
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmin ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1284,43 +1056,45 @@ define amdgpu_ps void @global_atomic_fmin_double_uni_address_div_value_agent_sco
define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-ITERATIVE-NEXT: br label [[TMP12]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[VAL:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL1:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP12:%.*]]
-; IR-DPP: 10:
-; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL:%.*]] syncscope("agent") monotonic, align 8
+; IR-DPP: 11:
+; IR-DPP-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[VAL]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP12]]
-; IR-DPP: 12:
-; IR-DPP-NEXT: br label [[TMP13]]
; IR-DPP: 13:
+; IR-DPP-NEXT: br label [[TMP13]]
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1329,73 +1103,45 @@ define amdgpu_ps void @global_atomic_fmax_double_uni_address_uni_value_agent_sco
define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(ptr addrspace(1) inreg %ptr, double %val) #1{
; IR-ITERATIVE-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] syncscope("agent") monotonic, align 8
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] syncscope("agent") monotonic, align 8
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ +qnan, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.maxnum.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fmax_double_uni_address_div_value_agent_scope_unsafe_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double +qnan) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP9]], double [[TMP10]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP11]], double [[TMP12]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP13]], double [[TMP14]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP15]], double [[TMP16]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP17]], double [[TMP18]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double +qnan, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.maxnum.f64(double [[TMP19]], double [[TMP20]], metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fmax.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fmax ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] syncscope("agent") monotonic, align 8
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("agent") monotonic
@@ -1404,51 +1150,45 @@ define amdgpu_ps void @global_atomic_fmax_double_uni_address_div_value_agent_sco
define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double inreg %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-ITERATIVE-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-ITERATIVE: 14:
+; IR-ITERATIVE: 11:
; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-ITERATIVE-NEXT: br label [[TMP16]]
-; IR-ITERATIVE: 16:
+; IR-ITERATIVE: 13:
; IR-ITERATIVE-NEXT: br label [[TMP17]]
-; IR-ITERATIVE: 17:
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_uni_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP17:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP3]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = trunc i64 [[TMP9]] to i32
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.uitofp.f64.i32(i32 [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.experimental.constrained.fmul.f64(double [[VAL:%.*]], double [[TMP11]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP13]], label [[TMP14:%.*]], label [[TMP16:%.*]]
-; IR-DPP: 14:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP15:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP12]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP16]]
-; IR-DPP: 16:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP17]]
-; IR-DPP: 17:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4
@@ -1457,73 +1197,45 @@ define amdgpu_ps void @global_atomic_fadd_double_uni_address_uni_value_system_sc
define amdgpu_ps void @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(ptr addrspace(1) inreg %ptr, double %val) #2 {
; IR-ITERATIVE-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-ITERATIVE-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP13:%.*]]
; IR-ITERATIVE: 2:
-; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
+; IR-ITERATIVE-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-ITERATIVE-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-ITERATIVE-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-ITERATIVE-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: br label [[COMPUTELOOP:%.*]]
-; IR-ITERATIVE: 10:
-; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17:%.*]] monotonic, align 4
-; IR-ITERATIVE-NEXT: br label [[TMP12:%.*]]
-; IR-ITERATIVE: 12:
-; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP17:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 1) #[[ATTR2]]
+; IR-ITERATIVE-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP8]], 0
+; IR-ITERATIVE-NEXT: br i1 [[TMP10]], label [[TMP14:%.*]], label [[TMP12:%.*]]
+; IR-ITERATIVE: 11:
+; IR-ITERATIVE-NEXT: [[TMP11:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP17]] monotonic, align 4
+; IR-ITERATIVE-NEXT: br label [[TMP12]]
; IR-ITERATIVE: 13:
+; IR-ITERATIVE-NEXT: br label [[TMP13]]
+; IR-ITERATIVE: 14:
; IR-ITERATIVE-NEXT: ret void
-; IR-ITERATIVE: ComputeLoop:
-; IR-ITERATIVE-NEXT: [[ACCUMULATOR:%.*]] = phi double [ -0.000000e+00, [[TMP2]] ], [ [[TMP17]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[ACTIVEBITS:%.*]] = phi i64 [ [[TMP9]], [[TMP2]] ], [ [[TMP20:%.*]], [[COMPUTELOOP]] ]
-; IR-ITERATIVE-NEXT: [[TMP14:%.*]] = call i64 @llvm.cttz.i64(i64 [[ACTIVEBITS]], i1 true) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
-; IR-ITERATIVE-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL:%.*]], i32 [[TMP15]]) #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP17]] = call double @llvm.experimental.constrained.fadd.f64(double [[ACCUMULATOR]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR7]]
-; IR-ITERATIVE-NEXT: [[TMP18:%.*]] = shl i64 1, [[TMP14]]
-; IR-ITERATIVE-NEXT: [[TMP19:%.*]] = xor i64 [[TMP18]], -1
-; IR-ITERATIVE-NEXT: [[TMP20]] = and i64 [[ACTIVEBITS]], [[TMP19]]
-; IR-ITERATIVE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[TMP20]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP21]], label [[COMPUTEEND:%.*]], label [[COMPUTELOOP]]
-; IR-ITERATIVE: ComputeEnd:
-; IR-ITERATIVE-NEXT: [[TMP22:%.*]] = icmp eq i32 [[TMP8]], 0
-; IR-ITERATIVE-NEXT: br i1 [[TMP22]], label [[TMP10:%.*]], label [[TMP12]]
;
; IR-DPP-LABEL: @global_atomic_fadd_double_uni_address_div_value_system_scope_strictfp(
-; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP1:%.*]] = call i1 @llvm.amdgcn.ps.live() #[[ATTR2]]
; IR-DPP-NEXT: br i1 [[TMP1]], label [[TMP2:%.*]], label [[TMP28:%.*]]
; IR-DPP: 2:
-; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
; IR-DPP-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP3]], 32
; IR-DPP-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
-; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP9:%.*]] = call double @llvm.amdgcn.set.inactive.f64(double [[VAL:%.*]], double -0.000000e+00) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP10:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP9]], i32 273, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP11:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP9]], double [[TMP10]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP12:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP11]], i32 274, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP13:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP11]], double [[TMP12]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP14:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP13]], i32 276, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP15:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP13]], double [[TMP14]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP16:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP15]], i32 280, i32 15, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP17:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP15]], double [[TMP16]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP18:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP17]], i32 322, i32 10, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP19:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP17]], double [[TMP18]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP20:%.*]] = call double @llvm.amdgcn.update.dpp.f64(double -0.000000e+00, double [[TMP19]], i32 323, i32 12, i32 15, i1 false) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP21:%.*]] = call double @llvm.experimental.constrained.fadd.f64(double [[TMP19]], double [[TMP20]], metadata !"round.dynamic", metadata !"fpexcept.strict") #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP22:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[TMP21]], i32 63) #[[ATTR8]]
-; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.strict.wwm.f64(double [[TMP22]]) #[[ATTR8]]
+; IR-DPP-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP4]], i32 0) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP8:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP6]], i32 [[TMP7]]) #[[ATTR2]]
+; IR-DPP-NEXT: [[TMP23:%.*]] = call double @llvm.amdgcn.wave.reduce.fadd.f64(double [[VAL:%.*]], i32 2) #[[ATTR2]]
; IR-DPP-NEXT: [[TMP24:%.*]] = icmp eq i32 [[TMP8]], 0
; IR-DPP-NEXT: br i1 [[TMP24]], label [[TMP25:%.*]], label [[TMP27:%.*]]
-; IR-DPP: 25:
+; IR-DPP: 11:
; IR-DPP-NEXT: [[TMP26:%.*]] = atomicrmw fadd ptr addrspace(1) [[PTR:%.*]], double [[TMP23]] monotonic, align 4
; IR-DPP-NEXT: br label [[TMP27]]
-; IR-DPP: 27:
+; IR-DPP: 13:
; IR-DPP-NEXT: br label [[TMP28]]
-; IR-DPP: 28:
+; IR-DPP: 14:
; IR-DPP-NEXT: ret void
;
%result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 7acf8067c2c25..d40deaefec300 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -18,34 +18,36 @@ declare double @div.double.value()
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_madmk_f32 v0, v2, 0x40800000, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-NEXT: .LBB0_3:
@@ -53,26 +55,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_madmk_f32 v0, v2, 0x40800000, v1
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -84,26 +88,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_mad_f32 v0, 4.0, v2, v1
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -115,30 +121,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_mad_f32 v0, 4.0, v2, v1
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-NEXT: .LBB0_3:
; GFX1032-NEXT: s_endpgm
@@ -146,73 +154,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB0_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB0_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_madmk_f32 v0, v2, 0x40800000, v1
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-DPP-NEXT: .LBB0_3:
@@ -220,26 +233,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_madmk_f32 v0, v2, 0x40800000, v1
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -251,26 +266,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -282,30 +299,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_mad_f32 v0, 4.0, v2, v1
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-DPP-NEXT: .LBB0_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -313,40 +332,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB0_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB0_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !atomic.ignore.denormal.mode !1
@@ -385,24 +407,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -414,15 +434,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB1_4
; GFX7LESS-NEXT: .LBB1_5:
@@ -458,34 +478,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -525,34 +545,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -592,38 +611,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB1_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB1_4
; GFX1032-NEXT: .LBB1_5:
; GFX1032-NEXT: s_endpgm
@@ -648,31 +666,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -697,30 +715,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -810,30 +826,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -890,44 +895,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -968,43 +971,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1032-DPP-NEXT: .LBB1_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -1030,46 +1027,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1095,39 +1090,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1138,45 +1124,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-NEXT: .LBB2_3:
@@ -1184,37 +1161,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1227,34 +1195,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1267,136 +1228,110 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-NEXT: .LBB2_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB2_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB2_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB2_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-DPP-NEXT: .LBB2_3:
@@ -1404,37 +1339,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1447,34 +1373,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1487,92 +1406,75 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-DPP-NEXT: .LBB2_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB2_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB2_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") monotonic, !amdgpu.no.fine.grained.memory !1, !atomic.ignore.denormal.mode !1
@@ -1612,24 +1514,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -1641,15 +1541,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB3_4
; GFX7LESS-NEXT: .LBB3_5:
@@ -1685,34 +1585,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1752,34 +1652,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1819,38 +1718,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB3_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB3_4
; GFX1032-NEXT: .LBB3_5:
; GFX1032-NEXT: s_endpgm
@@ -1875,31 +1773,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1924,30 +1822,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2037,30 +1933,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2117,44 +2002,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2195,43 +2078,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1032-DPP-NEXT: .LBB3_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -2257,46 +2134,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2322,39 +2197,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value() strictfp
@@ -2365,45 +2231,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-NEXT: .LBB4_3:
@@ -2411,37 +2268,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2454,34 +2302,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -2494,75 +2335,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-NEXT: .LBB4_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2575,85 +2404,71 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-NEXT: .LBB4_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-DPP-NEXT: .LBB4_3:
@@ -2661,37 +2476,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2704,34 +2510,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2744,75 +2543,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-DPP-NEXT: .LBB4_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2825,40 +2612,35 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-DPP-NEXT: .LBB4_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -2899,24 +2681,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -2928,15 +2708,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB5_4
; GFX7LESS-NEXT: .LBB5_5:
@@ -2972,34 +2752,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3039,34 +2819,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3106,38 +2885,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB5_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB5_4
; GFX1032-NEXT: .LBB5_5:
; GFX1032-NEXT: s_endpgm
@@ -3162,31 +2940,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -3211,30 +2989,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -3324,30 +3100,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -3404,44 +3169,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3482,43 +3245,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1032-DPP-NEXT: .LBB5_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -3544,46 +3301,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -3609,39 +3364,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -3682,24 +3428,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB6_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3711,15 +3455,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_4
; GFX7LESS-NEXT: .LBB6_5:
@@ -3755,34 +3499,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB6_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3822,34 +3566,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB6_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3889,38 +3632,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB6_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB6_4
; GFX1032-NEXT: .LBB6_5:
; GFX1032-NEXT: s_endpgm
@@ -3945,31 +3687,31 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v1, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v1, s3, v1
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_brev_b32 s2, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB6_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB6_4:
; GFX1164-NEXT: s_endpgm
;
@@ -3994,30 +3736,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v1, s2, v1
-; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_brev_b32 s0, 1
+; GFX1132-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB6_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB6_4:
; GFX1132-NEXT: s_endpgm
;
@@ -4107,30 +3847,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -4187,44 +3916,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4265,43 +3992,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1032-DPP-NEXT: .LBB6_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -4327,46 +4048,44 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB6_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -4392,39 +4111,30 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB6_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value() strictfp
@@ -4435,45 +4145,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: .LBB7_3:
@@ -4481,37 +4182,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4524,34 +4216,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4564,75 +4249,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-NEXT: .LBB7_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4645,85 +4318,71 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-NEXT: .LBB7_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-DPP-NEXT: .LBB7_3:
@@ -4731,37 +4390,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4774,34 +4424,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4814,75 +4457,63 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-DPP-NEXT: .LBB7_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4895,40 +4526,35 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4968,24 +4594,22 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB8_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB8_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -4997,15 +4621,15 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX7LESS-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_4
; GFX7LESS-NEXT: .LBB8_5:
@@ -5041,34 +4665,34 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB8_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5108,34 +4732,33 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB8_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -5175,38 +4798,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB8_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB8_4
; GFX1032-NEXT: .LBB8_5:
; GFX1032-NEXT: s_endpgm
@@ -5231,37 +4853,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB8_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -5293,42 +4914,40 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB8_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB8_4
; GFX1132-NEXT: .LBB8_5:
; GFX1132-NEXT: s_endpgm
@@ -5419,30 +5038,19 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -5499,44 +5107,42 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5577,43 +5183,37 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1032-DPP-NEXT: .LBB8_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -5639,52 +5239,48 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -5717,51 +5313,41 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1132-DPP-NEXT: .LBB8_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5773,39 +5359,41 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-NEXT: .LBB9_3:
@@ -5813,101 +5401,107 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-NEXT: .LBB9_3:
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-NEXT: .LBB9_3:
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-NEXT: .LBB9_3:
; GFX1032-NEXT: s_endpgm
@@ -5915,37 +5509,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-NEXT: .LBB9_3:
; GFX1164-NEXT: s_endpgm
@@ -5953,73 +5550,77 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-NEXT: .LBB9_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-DPP-NEXT: .LBB9_3:
@@ -6027,101 +5628,107 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
;
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-DPP-NEXT: .LBB9_3:
; GFX1064-DPP-NEXT: s_endpgm
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-DPP-NEXT: .LBB9_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6129,37 +5736,40 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-DPP-NEXT: .LBB9_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -6167,34 +5777,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6234,50 +5846,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB10_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB10_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_4
; GFX7LESS-NEXT: .LBB10_5:
; GFX7LESS-NEXT: s_endpgm
@@ -6312,42 +5924,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_4
; GFX9-NEXT: .LBB10_5:
; GFX9-NEXT: s_endpgm
@@ -6382,42 +5996,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB10_4
; GFX1064-NEXT: .LBB10_5:
; GFX1064-NEXT: s_endpgm
@@ -6452,41 +6066,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB10_4
; GFX1032-NEXT: .LBB10_5:
; GFX1032-NEXT: s_endpgm
@@ -6511,46 +6125,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB10_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB10_4
; GFX1164-NEXT: .LBB10_5:
; GFX1164-NEXT: s_endpgm
@@ -6576,44 +6190,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB10_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB10_4
; GFX1132-NEXT: .LBB10_5:
; GFX1132-NEXT: s_endpgm
@@ -6704,70 +6318,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX9-DPP-NEXT: .LBB10_3:
@@ -6804,61 +6420,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1064-DPP-NEXT: .LBB10_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -6893,56 +6517,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1032-DPP-NEXT: .LBB10_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6968,72 +6594,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1164-DPP-NEXT: .LBB10_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -7059,64 +6699,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1132-DPP-NEXT: .LBB10_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7128,48 +6768,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-NEXT: .LBB11_3:
@@ -7177,43 +6810,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-NEXT: .LBB11_3:
; GFX9-NEXT: s_endpgm
@@ -7221,40 +6847,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-NEXT: .LBB11_3:
; GFX1064-NEXT: s_endpgm
@@ -7262,171 +6883,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-NEXT: .LBB11_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-NEXT: .LBB11_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-NEXT: .LBB11_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-DPP-NEXT: .LBB11_3:
@@ -7434,43 +7037,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -7478,40 +7074,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-DPP-NEXT: .LBB11_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7519,123 +7110,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-DPP-NEXT: .LBB11_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-DPP-NEXT: .LBB11_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7675,50 +7255,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB12_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB12_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_4
; GFX7LESS-NEXT: .LBB12_5:
; GFX7LESS-NEXT: s_endpgm
@@ -7753,42 +7333,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB12_4
; GFX9-NEXT: .LBB12_5:
; GFX9-NEXT: s_endpgm
@@ -7823,42 +7405,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_4
; GFX1064-NEXT: .LBB12_5:
; GFX1064-NEXT: s_endpgm
@@ -7893,41 +7475,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB12_4
; GFX1032-NEXT: .LBB12_5:
; GFX1032-NEXT: s_endpgm
@@ -7952,46 +7534,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB12_4
; GFX1164-NEXT: .LBB12_5:
; GFX1164-NEXT: s_endpgm
@@ -8017,44 +7599,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB12_4
; GFX1132-NEXT: .LBB12_5:
; GFX1132-NEXT: s_endpgm
@@ -8145,70 +7727,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-DPP-NEXT: .LBB12_3:
@@ -8245,61 +7829,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-DPP-NEXT: .LBB12_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8334,56 +7926,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1032-DPP-NEXT: .LBB12_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -8409,72 +8003,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-DPP-NEXT: .LBB12_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -8500,64 +8108,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1132-DPP-NEXT: .LBB12_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8569,48 +8177,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: .LBB13_3:
@@ -8618,43 +8219,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: .LBB13_3:
; GFX9-NEXT: s_endpgm
@@ -8662,40 +8256,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: .LBB13_3:
; GFX1064-NEXT: s_endpgm
@@ -8703,171 +8292,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-NEXT: .LBB13_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-NEXT: .LBB13_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-NEXT: .LBB13_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-DPP-NEXT: .LBB13_3:
@@ -8875,43 +8446,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-DPP-NEXT: .LBB13_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -8919,40 +8483,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-DPP-NEXT: .LBB13_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8960,123 +8519,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-DPP-NEXT: .LBB13_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-DPP-NEXT: .LBB13_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-DPP-NEXT: .LBB13_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -9116,50 +8664,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB14_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB14_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB14_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_4
; GFX7LESS-NEXT: .LBB14_5:
; GFX7LESS-NEXT: s_endpgm
@@ -9194,42 +8742,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB14_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB14_4
; GFX9-NEXT: .LBB14_5:
; GFX9-NEXT: s_endpgm
@@ -9264,42 +8814,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB14_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_4
; GFX1064-NEXT: .LBB14_5:
; GFX1064-NEXT: s_endpgm
@@ -9334,41 +8884,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB14_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB14_4
; GFX1032-NEXT: .LBB14_5:
; GFX1032-NEXT: s_endpgm
@@ -9393,46 +8943,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB14_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_4
; GFX1164-NEXT: .LBB14_5:
; GFX1164-NEXT: s_endpgm
@@ -9458,44 +9008,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB14_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB14_4
; GFX1132-NEXT: .LBB14_5:
; GFX1132-NEXT: s_endpgm
@@ -9586,70 +9136,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX9-DPP-NEXT: .LBB14_3:
@@ -9686,61 +9238,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1064-DPP-NEXT: .LBB14_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -9775,56 +9335,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1032-DPP-NEXT: .LBB14_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -9850,72 +9412,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1164-DPP-NEXT: .LBB14_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -9941,64 +9517,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1132-DPP-NEXT: .LBB14_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10039,50 +9615,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB15_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB15_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB15_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_4
; GFX7LESS-NEXT: .LBB15_5:
; GFX7LESS-NEXT: s_endpgm
@@ -10117,42 +9693,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB15_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB15_4
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: s_endpgm
@@ -10187,42 +9765,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB15_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_4
; GFX1064-NEXT: .LBB15_5:
; GFX1064-NEXT: s_endpgm
@@ -10257,41 +9835,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB15_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB15_4
; GFX1032-NEXT: .LBB15_5:
; GFX1032-NEXT: s_endpgm
@@ -10316,46 +9894,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB15_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB15_4
; GFX1164-NEXT: .LBB15_5:
; GFX1164-NEXT: s_endpgm
@@ -10381,44 +9959,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB15_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB15_4
; GFX1132-NEXT: .LBB15_5:
; GFX1132-NEXT: s_endpgm
@@ -10509,70 +10087,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-DPP-NEXT: .LBB15_3:
@@ -10609,61 +10189,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-DPP-NEXT: .LBB15_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -10698,56 +10286,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1032-DPP-NEXT: .LBB15_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -10773,72 +10363,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-DPP-NEXT: .LBB15_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -10864,64 +10468,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1132-DPP-NEXT: .LBB15_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10933,48 +10537,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: .LBB16_3:
@@ -10982,43 +10579,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX9-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: .LBB16_3:
; GFX9-NEXT: s_endpgm
@@ -11026,40 +10616,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1064-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: .LBB16_3:
; GFX1064-NEXT: s_endpgm
@@ -11067,171 +10652,153 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-NEXT: .LBB16_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-NEXT: .LBB16_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-NEXT: .LBB16_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-DPP-NEXT: .LBB16_3:
@@ -11239,43 +10806,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX9-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-DPP-NEXT: .LBB16_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -11283,40 +10843,35 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1064-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-DPP-NEXT: .LBB16_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11324,123 +10879,112 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-DPP-NEXT: .LBB16_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-DPP-NEXT: .LBB16_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-DPP-NEXT: .LBB16_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11480,50 +11024,50 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB17_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB17_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_4
; GFX7LESS-NEXT: .LBB17_5:
; GFX7LESS-NEXT: s_endpgm
@@ -11558,42 +11102,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB17_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB17_4
; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: s_endpgm
@@ -11628,42 +11174,42 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB17_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_4
; GFX1064-NEXT: .LBB17_5:
; GFX1064-NEXT: s_endpgm
@@ -11698,41 +11244,41 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB17_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB17_4
; GFX1032-NEXT: .LBB17_5:
; GFX1032-NEXT: s_endpgm
@@ -11757,46 +11303,46 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB17_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_4
; GFX1164-NEXT: .LBB17_5:
; GFX1164-NEXT: s_endpgm
@@ -11822,44 +11368,44 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB17_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB17_4
; GFX1132-NEXT: .LBB17_5:
; GFX1132-NEXT: s_endpgm
@@ -11950,70 +11496,72 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX9-DPP-NEXT: .LBB17_3:
@@ -12050,61 +11598,69 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1064-DPP-NEXT: .LBB17_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -12139,56 +11695,58 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1032-DPP-NEXT: .LBB17_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12214,72 +11772,86 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1164-DPP-NEXT: .LBB17_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -12305,64 +11877,64 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1132-DPP-NEXT: .LBB17_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -12374,35 +11946,36 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB18_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB18_2
; GFX7LESS-NEXT: .LBB18_3:
@@ -12410,27 +11983,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB18_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12442,27 +12016,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB18_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -12474,31 +12049,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB18_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB18_2
; GFX1032-NEXT: .LBB18_3:
; GFX1032-NEXT: s_endpgm
@@ -12506,74 +12082,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB18_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB18_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB18_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB18_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB18_2
; GFX7LESS-DPP-NEXT: .LBB18_3:
@@ -12581,27 +12161,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12613,27 +12194,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -12645,31 +12227,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB18_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB18_2
; GFX1032-DPP-NEXT: .LBB18_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12677,40 +12260,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB18_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB18_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB18_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB18_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 monotonic, align 8, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1, !atomic.ignore.denormal.mode !1
@@ -12720,35 +12306,36 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
; GFX7LESS-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB19_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB19_2
; GFX7LESS-NEXT: .LBB19_3:
@@ -12756,27 +12343,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB19_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12788,27 +12376,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB19_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -12820,31 +12409,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB19_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB19_2
; GFX1032-NEXT: .LBB19_3:
; GFX1032-NEXT: s_endpgm
@@ -12852,74 +12442,78 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB19_2
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-NEXT: .LBB19_2:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB19_2
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-NEXT: .LBB19_2:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_add_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB19_2
; GFX7LESS-DPP-NEXT: .LBB19_3:
@@ -12927,27 +12521,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX9-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -12959,27 +12554,28 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1064-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -12991,31 +12587,32 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB19_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB19_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB19_2
; GFX1032-DPP-NEXT: .LBB19_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -13023,40 +12620,43 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_system_scope
; GFX1164-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB19_2
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1164-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1164-DPP-NEXT: .LBB19_2:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB19_2
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_add_f32 v1, v0, s[2:3]
+; GFX1132-DPP-NEXT: global_atomic_add_f32 v0, v1, s[2:3]
; GFX1132-DPP-NEXT: .LBB19_2:
; GFX1132-DPP-NEXT: s_endpgm
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 monotonic, align 4, !amdgpu.no.fine.grained.memory !1, !amdgpu.no.remote.memory !1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 0be764845cbb7..6adfd1dcc6958 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -43,7 +43,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -51,8 +52,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -152,7 +153,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -160,8 +162,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -269,33 +271,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB1_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -329,38 +328,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -400,30 +397,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB1_4:
; GFX1064-NEXT: s_endpgm
;
@@ -457,29 +452,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB1_4:
; GFX1032-NEXT: s_endpgm
;
@@ -503,34 +496,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -555,32 +545,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -653,37 +639,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -693,14 +661,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -742,46 +710,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB1_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -816,38 +776,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB1_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -872,56 +822,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -947,45 +885,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1021,7 +944,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1029,8 +953,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1130,7 +1054,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1138,8 +1063,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1248,33 +1173,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB3_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -1308,38 +1230,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1379,30 +1299,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB3_4:
; GFX1064-NEXT: s_endpgm
;
@@ -1436,29 +1354,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB3_4:
; GFX1032-NEXT: s_endpgm
;
@@ -1482,34 +1398,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1534,32 +1447,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -1632,37 +1541,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -1672,14 +1563,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1721,46 +1612,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB3_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -1795,38 +1678,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB3_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -1851,56 +1724,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1926,45 +1787,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2001,7 +1847,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2009,8 +1856,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2110,7 +1957,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -2118,8 +1966,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2227,33 +2075,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_max_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB5_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -2287,38 +2132,36 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_max_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2358,30 +2201,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmax v1, v0, s[0:1]
; GFX1064-NEXT: .LBB5_4:
; GFX1064-NEXT: s_endpgm
;
@@ -2415,29 +2256,27 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_max_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-NEXT: .LBB5_4:
; GFX1032-NEXT: s_endpgm
;
@@ -2461,34 +2300,31 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_max_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_max_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -2513,32 +2349,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_max_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_max_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_max_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2611,37 +2443,19 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2651,14 +2465,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2700,46 +2514,38 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB5_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -2774,38 +2580,28 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_max_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB5_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -2830,56 +2626,44 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_max_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2905,45 +2689,30 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_max_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_max_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2979,8 +2748,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-NEXT: s_cbranch_execz .LBB6_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2989,8 +2759,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -3044,7 +2814,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -3055,8 +2826,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -3078,7 +2849,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -3087,8 +2859,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3126,8 +2898,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3136,8 +2909,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3191,7 +2964,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3202,8 +2976,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3225,7 +2999,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3234,8 +3009,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3281,35 +3056,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB7_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -3343,46 +3118,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB7_4
; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: s_endpgm
@@ -3417,32 +3192,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB7_4:
; GFX1064-NEXT: s_endpgm
;
@@ -3476,31 +3251,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB7_4:
; GFX1032-NEXT: s_endpgm
;
@@ -3524,52 +3299,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB7_4
; GFX1164-NEXT: .LBB7_5:
; GFX1164-NEXT: s_endpgm
@@ -3595,49 +3367,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB7_4
; GFX1132-NEXT: .LBB7_5:
; GFX1132-NEXT: s_endpgm
@@ -3708,80 +3478,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX9-DPP-NEXT: .LBB7_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -3817,57 +3581,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB7_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -3901,50 +3667,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB7_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -3969,82 +3733,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -4074,56 +3841,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4136,9 +3898,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4175,8 +3937,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-NEXT: s_cbranch_execz .LBB8_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -4185,8 +3948,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -4240,7 +4003,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4251,8 +4015,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -4274,7 +4038,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4283,8 +4048,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4322,8 +4087,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4332,8 +4098,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4387,7 +4153,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4398,8 +4165,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4421,7 +4188,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4430,8 +4198,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4477,35 +4245,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB9_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB9_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB9_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -4539,46 +4307,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB9_4
; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: s_endpgm
@@ -4613,32 +4381,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB9_4:
; GFX1064-NEXT: s_endpgm
;
@@ -4672,31 +4440,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB9_4:
; GFX1032-NEXT: s_endpgm
;
@@ -4720,52 +4488,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_4
; GFX1164-NEXT: .LBB9_5:
; GFX1164-NEXT: s_endpgm
@@ -4791,49 +4556,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB9_4
; GFX1132-NEXT: .LBB9_5:
; GFX1132-NEXT: s_endpgm
@@ -4904,80 +4667,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -5013,57 +4770,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB9_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -5097,50 +4856,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB9_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -5165,82 +4922,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -5270,56 +5030,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
@@ -5332,9 +5087,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5371,8 +5126,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-NEXT: s_cbranch_execz .LBB10_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -5381,8 +5137,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -5436,7 +5192,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -5447,8 +5204,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -5470,7 +5227,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -5479,8 +5237,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5518,8 +5276,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5528,8 +5287,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5583,7 +5342,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5594,8 +5354,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5617,7 +5377,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5626,8 +5387,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5673,35 +5434,35 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmax_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB11_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -5735,46 +5496,46 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_max_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB11_4
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: s_endpgm
@@ -5809,32 +5570,32 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB11_4:
; GFX1064-NEXT: s_endpgm
;
@@ -5868,31 +5629,31 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmax_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB11_4:
; GFX1032-NEXT: s_endpgm
;
@@ -5916,52 +5677,49 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_4
; GFX1164-NEXT: .LBB11_5:
; GFX1164-NEXT: s_endpgm
@@ -5987,49 +5745,47 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_max_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_max_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB11_4
; GFX1132-NEXT: .LBB11_5:
; GFX1132-NEXT: s_endpgm
@@ -6100,80 +5856,74 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -6209,57 +5959,59 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB11_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -6293,50 +6045,48 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmax_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB11_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -6361,82 +6111,85 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -6466,56 +6219,51 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
@@ -6528,9 +6276,9 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6567,7 +6315,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6575,8 +6324,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6676,7 +6425,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6684,8 +6434,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6789,7 +6539,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6797,8 +6548,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6898,7 +6649,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6906,8 +6658,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 2a3f7d71e72af..e4d5ce2741ad6 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -43,7 +43,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -51,8 +52,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -152,7 +153,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -160,8 +162,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -269,33 +271,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB1_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -329,38 +328,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -400,30 +397,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB1_4:
; GFX1064-NEXT: s_endpgm
;
@@ -457,29 +452,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB1_4:
; GFX1032-NEXT: s_endpgm
;
@@ -503,34 +496,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB1_4:
; GFX1164-NEXT: s_endpgm
;
@@ -555,32 +545,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB1_4:
; GFX1132-NEXT: s_endpgm
;
@@ -653,37 +639,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -693,14 +661,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -742,46 +710,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB1_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -816,38 +776,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB1_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -872,56 +822,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB1_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -947,45 +885,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB1_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -1021,7 +944,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1029,8 +953,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1130,7 +1054,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -1138,8 +1063,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1248,33 +1173,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB3_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -1308,38 +1230,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1379,30 +1299,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB3_4:
; GFX1064-NEXT: s_endpgm
;
@@ -1436,29 +1354,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB3_4:
; GFX1032-NEXT: s_endpgm
;
@@ -1482,34 +1398,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB3_4:
; GFX1164-NEXT: s_endpgm
;
@@ -1534,32 +1447,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB3_4:
; GFX1132-NEXT: s_endpgm
;
@@ -1632,37 +1541,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -1672,14 +1563,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1721,46 +1612,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB3_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -1795,38 +1678,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB3_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -1851,56 +1724,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB3_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -1926,45 +1787,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB3_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2001,7 +1847,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2009,8 +1856,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2110,7 +1957,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -2118,8 +1966,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2227,33 +2075,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s4
-; GFX7LESS-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s3
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX7LESS-NEXT: v_min_f32_e32 v2, s2, v2
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s2, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_4
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin v1, off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin v0, off, s[0:3], 0
; GFX7LESS-NEXT: .LBB5_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -2287,38 +2132,36 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_mov_b32 s4, 0x7fc00000
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_readlane_b32 s3, v0, s2
-; GFX9-NEXT: v_max_f32_e64 v1, s3, s3
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f32_e32 v2, v2, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX9-NEXT: v_min_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX9-NEXT: v_max_f32_e64 v3, s4, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2358,30 +2201,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1064-NEXT: s_mov_b32 s2, 0x7fc00000
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s3, s[0:1]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_4
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: v_mov_b32_e32 v0, s2
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1064-NEXT: global_atomic_fmin v1, v0, s[0:1]
; GFX1064-NEXT: .LBB5_4:
; GFX1064-NEXT: s_endpgm
;
@@ -2415,29 +2256,27 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1032-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s2, s1
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1032-NEXT: s_bitset0_b32 s1, s2
+; GFX1032-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1032-NEXT: v_min_f32_e64 v2, s0, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB5_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin v0, v1, s[0:1]
+; GFX1032-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-NEXT: .LBB5_4:
; GFX1032-NEXT: s_endpgm
;
@@ -2461,34 +2300,31 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0x7fc00000
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
-; GFX1164-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f32_e64 v2, s3, s3
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_mov_b32 s2, 0x7fc00000
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_min_f32_e64 v2, s2, s4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_4
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-NEXT: v_mov_b32_e32 v0, s2
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1164-NEXT: global_atomic_min_f32 v1, v0, s[0:1]
; GFX1164-NEXT: .LBB5_4:
; GFX1164-NEXT: s_endpgm
;
@@ -2513,32 +2349,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f32_e64 v2, s2, s2
-; GFX1132-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: s_mov_b32 s0, 0x7fc00000
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1132-NEXT: s_bitset0_b32 s1, s2
+; GFX1132-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132-NEXT: v_min_f32_e64 v2, s0, s3
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_4
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_atomic_min_f32 v0, v1, s[0:1]
+; GFX1132-NEXT: global_atomic_min_f32 v1, v0, s[2:3]
; GFX1132-NEXT: .LBB5_4:
; GFX1132-NEXT: s_endpgm
;
@@ -2611,37 +2443,19 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v5, v5, v5
-; GFX9-DPP-NEXT: v_min_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX9-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2651,14 +2465,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
+; GFX9-DPP-NEXT: v_max_f32_e64 v4, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v4
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2700,46 +2514,38 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1064-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v5, v5
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 0
-; GFX1064-DPP-NEXT: v_max_f32_e64 v3, s2, s2
-; GFX1064-DPP-NEXT: v_max_f32_e64 v4, s3, s3
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_min_f32_e32 v3, v4, v3
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB5_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -2774,38 +2580,28 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
-; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v3
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7fc00000, v0, s0
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_min_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin v1, v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB5_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -2830,56 +2626,44 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b32 s0, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, s0
; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_min_f32 v4, v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB5_2:
; GFX1164-DPP-NEXT: s_endpgm
;
@@ -2905,45 +2689,30 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fc00000, v0, s0
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v2, v2 :: v_dual_max_f32 v1, v1, v1
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_max_f32 v2, v3, v3 :: v_dual_mov_b32 v3, 0x7fc00000
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, s1
+; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_atomic_min_f32 v4, v0, s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_min_f32 v3, v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB5_2:
; GFX1132-DPP-NEXT: s_endpgm
%divValue = call float @div.float.value()
@@ -2979,8 +2748,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-NEXT: s_cbranch_execz .LBB6_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2989,8 +2759,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -3044,7 +2814,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -3055,8 +2826,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -3078,7 +2849,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -3087,8 +2859,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3126,8 +2898,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3136,8 +2909,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX9-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3191,7 +2964,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3202,8 +2976,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -3225,7 +2999,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -3234,8 +3009,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -3281,35 +3056,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB7_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -3343,46 +3118,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB7_4
; GFX9-NEXT: .LBB7_5:
; GFX9-NEXT: s_endpgm
@@ -3417,32 +3192,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB7_4:
; GFX1064-NEXT: s_endpgm
;
@@ -3476,31 +3251,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB7_4:
; GFX1032-NEXT: s_endpgm
;
@@ -3524,52 +3299,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB7_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB7_4
; GFX1164-NEXT: .LBB7_5:
; GFX1164-NEXT: s_endpgm
@@ -3595,49 +3367,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB7_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB7_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB7_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB7_4
; GFX1132-NEXT: .LBB7_5:
; GFX1132-NEXT: s_endpgm
@@ -3708,80 +3478,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX9-DPP-NEXT: .LBB7_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -3817,57 +3581,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB7_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -3901,50 +3667,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB7_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -3969,82 +3733,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -4074,56 +3841,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
@@ -4136,9 +3898,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4175,8 +3937,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-NEXT: s_cbranch_execz .LBB8_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -4185,8 +3948,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -4240,7 +4003,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4251,8 +4015,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -4274,7 +4038,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -4283,8 +4048,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4322,8 +4087,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4332,8 +4098,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX9-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4387,7 +4153,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4398,8 +4165,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -4421,7 +4188,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -4430,8 +4198,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -4477,35 +4245,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB9_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB9_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB9_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -4539,46 +4307,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB9_4
; GFX9-NEXT: .LBB9_5:
; GFX9-NEXT: s_endpgm
@@ -4613,32 +4381,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB9_4:
; GFX1064-NEXT: s_endpgm
;
@@ -4672,31 +4440,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB9_4:
; GFX1032-NEXT: s_endpgm
;
@@ -4720,52 +4488,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_4
; GFX1164-NEXT: .LBB9_5:
; GFX1164-NEXT: s_endpgm
@@ -4791,49 +4556,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB9_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB9_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB9_4
; GFX1132-NEXT: .LBB9_5:
; GFX1132-NEXT: s_endpgm
@@ -4904,80 +4667,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -5013,57 +4770,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB9_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -5097,50 +4856,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB9_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -5165,82 +4922,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -5270,56 +5030,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
@@ -5332,9 +5087,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5371,8 +5126,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-NEXT: s_cbranch_execz .LBB10_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -5381,8 +5137,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
@@ -5436,7 +5192,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -5447,8 +5204,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
@@ -5470,7 +5227,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
@@ -5479,8 +5237,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5518,8 +5276,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5528,8 +5287,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5583,7 +5342,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5594,8 +5354,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
@@ -5617,7 +5377,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[4:5], 4.0, 4.0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -5626,8 +5387,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], 4.0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
@@ -5673,35 +5434,35 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX7LESS-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX7LESS-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s0, 0
+; GFX7LESS-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s5, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s1
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s0
+; GFX7LESS-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v3
+; GFX7LESS-NEXT: s_or_b64 s[4:5], s[2:3], s[2:3]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s1, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_4
; GFX7LESS-NEXT: ; %bb.3:
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[34:35], 0x9
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s1
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[2:3], off, s[0:3], 0
+; GFX7LESS-NEXT: buffer_atomic_fmin_x2 v[0:1], off, s[4:7], 0
; GFX7LESS-NEXT: .LBB11_4:
; GFX7LESS-NEXT: s_endpgm
;
@@ -5735,46 +5496,46 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX9-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX9-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_min_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB11_4
; GFX9-NEXT: .LBB11_5:
; GFX9-NEXT: s_endpgm
@@ -5809,32 +5570,32 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_4
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1064-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-NEXT: .LBB11_4:
; GFX1064-NEXT: s_endpgm
;
@@ -5868,31 +5629,31 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1032-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_4
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v0, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_atomic_fmin_x2 v0, v[2:3], s[0:1]
+; GFX1032-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-NEXT: .LBB11_4:
; GFX1032-NEXT: s_endpgm
;
@@ -5916,52 +5677,49 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[0:1], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[0:1]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_4
; GFX1164-NEXT: .LBB11_5:
; GFX1164-NEXT: s_endpgm
@@ -5987,49 +5745,47 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0x7ff80000
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB11_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_mov_b32 s1, 0x7ff80000
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_max_f64 v[4:5], s[2:3], s[2:3]
-; GFX1132-NEXT: v_min_f64 v[4:5], v[2:3], v[4:5]
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_min_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-NEXT: v_max_f64 v[4:5], s[0:1], s[0:1]
+; GFX1132-NEXT: s_mov_b32 s0, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[2:3]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX1132-NEXT: s_cbranch_execnz .LBB11_4
; GFX1132-NEXT: .LBB11_5:
; GFX1132-NEXT: s_endpgm
@@ -6100,80 +5856,74 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v8, 0x7ff80000
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[7:8], v[7:8], v[7:8]
-; GFX9-DPP-NEXT: v_min_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
+; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v2, s[0:1]
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f64 v[1:2], s[0:1], s[0:1]
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f64 v[9:10], v[11:12], v[11:12]
-; GFX9-DPP-NEXT: v_min_f64 v[9:10], v[9:10], v[1:2]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_max_f64 v[8:9], v[10:11], v[10:11]
+; GFX9-DPP-NEXT: v_min_f64 v[8:9], v[8:9], v[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[8:9], v2, v[8:11], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
-; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v8
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -6209,57 +5959,59 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_max_f64 v[3:4], s[4:5], s[4:5]
-; GFX1064-DPP-NEXT: v_max_f64 v[5:6], s[2:3], s[2:3]
-; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1064-DPP-NEXT: .LBB11_2:
; GFX1064-DPP-NEXT: s_endpgm
;
@@ -6293,50 +6045,48 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x7ff80000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
-; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[3:4]
-; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7ff80000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0x7ff80000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_atomic_fmin_x2 v2, v[0:1], s[2:3]
; GFX1032-DPP-NEXT: .LBB11_2:
; GFX1032-DPP-NEXT: s_endpgm
;
@@ -6361,82 +6111,85 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, 0x7ff80000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v11, 0
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v11, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[7:8], v[9:10], v[9:10]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_min_f64 v[7:8], v[7:8], v[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v11, v[7:10], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
@@ -6466,56 +6219,51 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x7ff80000, v1, s0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v5, 0x7ff80000 :: v_dual_mov_b32 v4, 0
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s3, v3, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v10, 0
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], s[2:3], s[2:3]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
@@ -6528,9 +6276,9 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6567,7 +6315,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6575,8 +6324,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6676,7 +6425,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6684,8 +6434,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6789,7 +6539,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -6797,8 +6548,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -6898,7 +6649,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v2, 4.0, 4.0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
@@ -6906,8 +6658,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 94476b93b4f28..f3a595d24deec 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -18,34 +18,36 @@ declare double @div.double.value()
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_madmk_f32 v0, v2, 0xc0800000, v1
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-NEXT: .LBB0_3:
@@ -53,26 +55,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_3
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_madmk_f32 v0, v2, 0xc0800000, v1
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -84,26 +88,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_mad_f32 v0, -4.0, v2, v1
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -115,60 +121,64 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_mad_f32 v0, -4.0, v2, v1
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-NEXT: .LBB0_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-NEXT: ; %bb.1:
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b32 s4, s[0:1], 0x0
+; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v1, s4
+; GFX1164-NEXT: v_mov_b32_e32 v1, s2
+; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -181,68 +191,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
-; GFX1132-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB0_2
; GFX1132-NEXT: .LBB0_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s6, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_madmk_f32 v0, v2, 0xc0800000, v1
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX7LESS-DPP-NEXT: .LBB0_3:
@@ -250,26 +263,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX9-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_madmk_f32 v0, v2, 0xc0800000, v1
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -281,26 +296,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -312,60 +329,64 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_mad_f32 v0, -4.0, v2, v1
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1032-DPP-NEXT: .LBB0_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1164-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
+; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -378,34 +399,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1132-DPP-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB0_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB0_2
; GFX1132-DPP-NEXT: .LBB0_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -445,24 +467,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB1_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -474,15 +494,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB1_4
; GFX7LESS-NEXT: .LBB1_5:
@@ -518,34 +538,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB1_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -585,34 +605,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB1_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -652,38 +671,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB1_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB1_4
; GFX1032-NEXT: .LBB1_5:
; GFX1032-NEXT: s_endpgm
@@ -708,37 +726,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB1_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -770,42 +787,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB1_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB1_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB1_4
; GFX1132-NEXT: .LBB1_5:
; GFX1132-NEXT: s_endpgm
@@ -896,30 +911,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -976,44 +980,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1054,43 +1056,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1032-DPP-NEXT: .LBB1_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -1116,52 +1112,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -1194,51 +1186,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB1_2
; GFX1132-DPP-NEXT: .LBB1_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -1250,45 +1232,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-NEXT: .LBB2_3:
@@ -1296,37 +1269,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1339,34 +1303,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1379,75 +1336,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-NEXT: .LBB2_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB2_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -1460,85 +1405,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB2_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB2_2
; GFX1132-NEXT: .LBB2_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX7LESS-DPP-NEXT: .LBB2_3:
@@ -1546,37 +1477,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1589,34 +1511,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1629,75 +1544,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1032-DPP-NEXT: .LBB2_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -1710,40 +1613,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB2_2
; GFX1132-DPP-NEXT: .LBB2_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -1784,24 +1682,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB3_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB3_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB3_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -1813,15 +1709,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX7LESS-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB3_4
; GFX7LESS-NEXT: .LBB3_5:
@@ -1857,34 +1753,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB3_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -1924,34 +1820,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB3_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -1991,38 +1886,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB3_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB3_4
; GFX1032-NEXT: .LBB3_5:
; GFX1032-NEXT: s_endpgm
@@ -2047,37 +1941,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB3_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2109,42 +2002,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB3_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB3_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB3_4
; GFX1132-NEXT: .LBB3_5:
; GFX1132-NEXT: s_endpgm
@@ -2235,30 +2126,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -2315,44 +2195,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2393,43 +2271,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1032-DPP-NEXT: .LBB3_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -2455,52 +2327,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -2533,51 +2401,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB3_2
; GFX1132-DPP-NEXT: .LBB3_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -2589,45 +2447,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-NEXT: .LBB4_3:
@@ -2635,37 +2484,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2678,34 +2518,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -2718,75 +2551,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-NEXT: .LBB4_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -2799,85 +2620,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-NEXT: .LBB4_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX7LESS-DPP-NEXT: .LBB4_3:
@@ -2885,37 +2692,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -2928,34 +2726,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -2968,75 +2759,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1032-DPP-NEXT: .LBB4_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3049,40 +2828,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB4_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB4_2
; GFX1132-DPP-NEXT: .LBB4_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -3123,24 +2897,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB5_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3152,15 +2924,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB5_4
; GFX7LESS-NEXT: .LBB5_5:
@@ -3196,34 +2968,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB5_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -3263,34 +3035,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB5_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -3330,38 +3101,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB5_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB5_4
; GFX1032-NEXT: .LBB5_5:
; GFX1032-NEXT: s_endpgm
@@ -3386,37 +3156,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB5_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -3448,42 +3217,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB5_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB5_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB5_4
; GFX1132-NEXT: .LBB5_5:
; GFX1132-NEXT: s_endpgm
@@ -3574,30 +3341,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -3654,44 +3410,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -3732,43 +3486,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1032-DPP-NEXT: .LBB5_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -3794,52 +3542,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -3872,51 +3616,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB5_2
; GFX1132-DPP-NEXT: .LBB5_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -3958,24 +3692,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB6_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB6_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -3987,15 +3719,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX7LESS-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_4
; GFX7LESS-NEXT: .LBB6_5:
@@ -4031,34 +3763,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB6_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4098,34 +3830,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB6_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4165,38 +3896,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB6_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB6_4
; GFX1032-NEXT: .LBB6_5:
; GFX1032-NEXT: s_endpgm
@@ -4221,37 +3951,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB6_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4283,42 +4012,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB6_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB6_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB6_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB6_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB6_4
; GFX1132-NEXT: .LBB6_5:
; GFX1132-NEXT: s_endpgm
@@ -4409,30 +4136,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -4489,44 +4205,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -4567,43 +4281,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1032-DPP-NEXT: .LBB6_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -4629,52 +4337,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -4707,51 +4411,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB6_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB6_2
; GFX1132-DPP-NEXT: .LBB6_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -4763,45 +4457,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: .LBB7_3:
@@ -4809,37 +4494,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX9-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -4852,34 +4528,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1064-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -4892,75 +4561,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-NEXT: .LBB7_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -4973,85 +4630,71 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s3, 0
+; GFX1132-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-NEXT: .LBB7_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-DPP-NEXT: .LBB7_3:
@@ -5059,37 +4702,28 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX9-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
-; GFX9-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX9-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5102,34 +4736,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1064-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_load_dword s2, s[0:1], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5142,75 +4769,63 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1032-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
-; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1032-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-DPP-NEXT: .LBB7_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_load_b32 s2, s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s2
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX1164-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v0, s6, v1
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5223,40 +4838,35 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
+; GFX1132-DPP-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f32_e32 v0, 4.0, v0
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b32 s3, s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b32 s4, s[0:1], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mul_f32 v2, 4.0, v0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, s4
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-DPP-NEXT: .LBB7_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -5296,24 +4906,22 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX7LESS-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_brev_b32 s6, 1
+; GFX7LESS-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s4, v0, s2
-; GFX7LESS-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, s4, v2
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB8_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX7LESS-NEXT: v_add_f32_e32 v2, s6, v2
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB8_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB8_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
@@ -5325,15 +4933,15 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX7LESS-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_sub_f32_e32 v0, v1, v2
+; GFX7LESS-NEXT: v_subrev_f32_e32 v0, s6, v1
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[3:4], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB8_4
; GFX7LESS-NEXT: .LBB8_5:
@@ -5369,34 +4977,34 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX9-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_brev_b32 s4, 1
+; GFX9-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_ff1_i32_b64 s2, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s4, v0, s2
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-NEXT: s_bitset0_b64 s[0:1], s2
; GFX9-NEXT: v_add_f32_e32 v2, s4, v2
+; GFX9-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX9-NEXT: v_readfirstlane_b32 s4, v2
; GFX9-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX9-NEXT: s_cbranch_execz .LBB8_5
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX9-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -5436,34 +5044,33 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_brev_b32 s4, 1
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1064-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1064-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1064-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v2
; GFX1064-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX1064-NEXT: s_cbranch_execz .LBB8_5
; GFX1064-NEXT: ; %bb.3:
; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1064-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1064-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-NEXT: v_mov_b32_e32 v1, v0
@@ -5503,38 +5110,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_brev_b32 s2, 1
+; GFX1032-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1032-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1032-NEXT: s_bitset0_b32 s0, s1
+; GFX1032-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1032-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v2
; GFX1032-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX1032-NEXT: s_cbranch_execz .LBB8_5
; GFX1032-NEXT: ; %bb.3:
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v2, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dword v1, v3, s[0:1]
+; GFX1032-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
+; GFX1032-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-NEXT: s_cbranch_execnz .LBB8_4
; GFX1032-NEXT: .LBB8_5:
; GFX1032-NEXT: s_endpgm
@@ -5559,37 +5165,36 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: s_brev_b32 s4, 1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX1164-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_ctz_i32_b64 s2, s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1164-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f32_e32 v2, s3, v2
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: s_bitset0_b64 s[0:1], s2
+; GFX1164-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164-NEXT: v_add_f32_e64 v2, s4, s3
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v2
; GFX1164-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1164-NEXT: ; %bb.2:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1164-NEXT: s_cbranch_execz .LBB8_5
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, 0
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1164-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1164-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1164-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1164-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1164-NEXT: v_mov_b32_e32 v1, v0
@@ -5621,42 +5226,40 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_bfrev_b32_e32 v2, 1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB8_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_brev_b32 s2, 1
+; GFX1132-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f32_e32 v2, s2, v2
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: v_readlane_b32 s3, v0, s1
+; GFX1132-NEXT: s_bitset0_b32 s0, s1
+; GFX1132-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1132-NEXT: v_add_f32_e64 v2, s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v2
; GFX1132-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
+; GFX1132-NEXT: ; %bb.2:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v1
; GFX1132-NEXT: s_cbranch_execz .LBB8_5
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, 0
+; GFX1132-NEXT: s_mov_b32 s3, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b32 v1, v3, s[0:1]
+; GFX1132-NEXT: global_load_b32 v1, v2, s[0:1]
; GFX1132-NEXT: .LBB8_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
+; GFX1132-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1132-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1132-NEXT: v_mov_b32_e32 v1, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-NEXT: s_cbranch_execnz .LBB8_4
; GFX1132-NEXT: .LBB8_5:
; GFX1132-NEXT: s_endpgm
@@ -5747,30 +5350,19 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, v3, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v3, v3, v0, s[0:1]
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v4 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v4 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX9-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_bcast:31 row_mask:0xf bank_mask:0xf
; GFX9-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
@@ -5827,44 +5419,42 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_permute_b32 v4, v5, v3
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v3, 32
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_add_f32_e64 v3, s2, s3
+; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 63
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1064-DPP-NEXT: ; %bb.1:
; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1064-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1064-DPP-NEXT: v_subrev_f32_e32 v0, s4, v1
+; GFX1064-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v0
@@ -5905,43 +5495,37 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v5
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v0, s0
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_add_f32_dpp v3, v3, v3 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v4, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX1032-DPP-NEXT: v_readlane_b32 s2, v3, 31
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s3, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1032-DPP-NEXT: ; %bb.1:
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dword v1, v6, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX1032-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v6, v[0:1], s[0:1] glc
+; GFX1032-DPP-NEXT: v_subrev_f32_e32 v0, s2, v1
+; GFX1032-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1032-DPP-NEXT: .LBB8_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -5967,52 +5551,48 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s[0:1]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v3, -1, 0
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v3, -1, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v3, 32, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v3, 4, v3
+; GFX1164-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1164-DPP-NEXT: ds_permute_b32 v2, v3, v1
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v2, v1
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1164-DPP-NEXT: v_readlane_b32 s4, v1, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, exec_hi, v0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b32 v5, v0, s[0:1]
; GFX1164-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1164-DPP-NEXT: v_subrev_f32_e32 v4, s4, v5
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b32 v4, v0, v[4:5], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v4
@@ -6045,51 +5625,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v1, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v2, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v1 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v1 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_add_f32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1132-DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s3, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b32 v5, v6, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b32 v4, v0, s[0:1]
; GFX1132-DPP-NEXT: .LBB8_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_sub_f32_e32 v4, v5, v0
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v4, v6, v[4:5], s[0:1] glc
+; GFX1132-DPP-NEXT: v_subrev_f32_e32 v3, s2, v4
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b32 v3, v0, v[3:4], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v5, v4
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, v3
+; GFX1132-DPP-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB8_2
; GFX1132-DPP-NEXT: .LBB8_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6101,39 +5671,41 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe(ptr addrspace(1) %ptr) #0 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-NEXT: .LBB9_3:
@@ -6141,101 +5713,107 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-NEXT: .LBB9_3:
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-NEXT: .LBB9_3:
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-NEXT: .LBB9_3:
; GFX1032-NEXT: s_endpgm
@@ -6243,37 +5821,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164: ; %bb.0:
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-NEXT: .LBB9_3:
; GFX1164-NEXT: s_endpgm
@@ -6281,73 +5862,77 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-NEXT: .LBB9_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX7LESS-DPP-NEXT: .LBB9_3:
@@ -6355,101 +5940,107 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX9-DPP-NEXT: .LBB9_3:
; GFX9-DPP-NEXT: s_endpgm
;
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1064-DPP: ; %bb.0:
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1064-DPP-NEXT: .LBB9_3:
; GFX1064-DPP-NEXT: s_endpgm
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1032-DPP-NEXT: .LBB9_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -6457,37 +6048,40 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1164-DPP: ; %bb.0:
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1164-DPP-NEXT: .LBB9_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -6495,34 +6089,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1132-DPP-NEXT: v_cvt_f64_u32_e32 v[0:1], s0
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], v[0:1], 4.0
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB9_2
; GFX1132-DPP-NEXT: .LBB9_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -6562,50 +6158,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB10_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB10_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_4
; GFX7LESS-NEXT: .LBB10_5:
; GFX7LESS-NEXT: s_endpgm
@@ -6640,42 +6236,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB10_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB10_4
; GFX9-NEXT: .LBB10_5:
; GFX9-NEXT: s_endpgm
@@ -6710,42 +6308,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB10_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB10_4
; GFX1064-NEXT: .LBB10_5:
; GFX1064-NEXT: s_endpgm
@@ -6780,41 +6378,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB10_4
; GFX1032-NEXT: .LBB10_5:
; GFX1032-NEXT: s_endpgm
@@ -6839,46 +6437,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB10_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB10_4
; GFX1164-NEXT: .LBB10_5:
; GFX1164-NEXT: s_endpgm
@@ -6904,44 +6502,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB10_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB10_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB10_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB10_4
; GFX1132-NEXT: .LBB10_5:
; GFX1132-NEXT: s_endpgm
@@ -7032,70 +6630,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX9-DPP-NEXT: .LBB10_3:
@@ -7132,61 +6732,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1064-DPP-NEXT: .LBB10_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7221,56 +6829,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1032-DPP-NEXT: .LBB10_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -7296,72 +6906,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1164-DPP-NEXT: .LBB10_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -7387,64 +7011,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB10_2
; GFX1132-DPP-NEXT: .LBB10_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -7456,48 +7080,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp(ptr addrspace(1) %ptr) #1 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-NEXT: .LBB11_3:
@@ -7505,43 +7122,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB11_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-NEXT: .LBB11_3:
; GFX9-NEXT: s_endpgm
@@ -7549,40 +7159,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-NEXT: .LBB11_3:
; GFX1064-NEXT: s_endpgm
@@ -7590,171 +7195,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-NEXT: .LBB11_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-NEXT: .LBB11_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-NEXT: .LBB11_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX7LESS-DPP-NEXT: .LBB11_3:
@@ -7762,43 +7349,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX9-DPP-NEXT: .LBB11_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -7806,40 +7386,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1064-DPP-NEXT: .LBB11_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -7847,123 +7422,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1032-DPP-NEXT: .LBB11_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1164-DPP-NEXT: .LBB11_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB11_2
; GFX1132-DPP-NEXT: .LBB11_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8002,50 +7566,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB12_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB12_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB12_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_4
; GFX7LESS-NEXT: .LBB12_5:
; GFX7LESS-NEXT: s_endpgm
@@ -8080,42 +7644,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB12_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB12_4
; GFX9-NEXT: .LBB12_5:
; GFX9-NEXT: s_endpgm
@@ -8150,42 +7716,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB12_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB12_4
; GFX1064-NEXT: .LBB12_5:
; GFX1064-NEXT: s_endpgm
@@ -8220,41 +7786,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB12_4
; GFX1032-NEXT: .LBB12_5:
; GFX1032-NEXT: s_endpgm
@@ -8279,46 +7845,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB12_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB12_4
; GFX1164-NEXT: .LBB12_5:
; GFX1164-NEXT: s_endpgm
@@ -8344,44 +7910,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB12_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB12_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB12_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB12_4
; GFX1132-NEXT: .LBB12_5:
; GFX1132-NEXT: s_endpgm
@@ -8472,70 +8038,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX9-DPP-NEXT: .LBB12_3:
@@ -8572,61 +8140,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1064-DPP-NEXT: .LBB12_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -8661,56 +8237,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1032-DPP-NEXT: .LBB12_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -8736,72 +8314,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1164-DPP-NEXT: .LBB12_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -8827,64 +8419,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB12_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB12_2
; GFX1132-DPP-NEXT: .LBB12_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -8896,48 +8488,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp(ptr addrspace(1) %ptr) #2{
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-NEXT: .LBB13_3:
@@ -8945,43 +8530,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB13_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-NEXT: .LBB13_3:
; GFX9-NEXT: s_endpgm
@@ -8989,40 +8567,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-NEXT: .LBB13_3:
; GFX1064-NEXT: s_endpgm
@@ -9030,171 +8603,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-NEXT: .LBB13_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-NEXT: .LBB13_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-NEXT: .LBB13_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX7LESS-DPP-NEXT: .LBB13_3:
@@ -9202,43 +8757,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX9-DPP-NEXT: .LBB13_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -9246,40 +8794,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1064-DPP-NEXT: .LBB13_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -9287,123 +8830,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1032-DPP-NEXT: .LBB13_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1164-DPP-NEXT: .LBB13_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB13_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB13_2
; GFX1132-DPP-NEXT: .LBB13_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -9443,50 +8975,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB14_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB14_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB14_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB14_4
; GFX7LESS-NEXT: .LBB14_5:
; GFX7LESS-NEXT: s_endpgm
@@ -9521,42 +9053,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB14_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB14_4
; GFX9-NEXT: .LBB14_5:
; GFX9-NEXT: s_endpgm
@@ -9591,42 +9125,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB14_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB14_4
; GFX1064-NEXT: .LBB14_5:
; GFX1064-NEXT: s_endpgm
@@ -9661,41 +9195,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB14_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB14_4
; GFX1032-NEXT: .LBB14_5:
; GFX1032-NEXT: s_endpgm
@@ -9720,46 +9254,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB14_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB14_4
; GFX1164-NEXT: .LBB14_5:
; GFX1164-NEXT: s_endpgm
@@ -9785,44 +9319,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB14_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB14_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB14_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB14_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB14_4
; GFX1132-NEXT: .LBB14_5:
; GFX1132-NEXT: s_endpgm
@@ -9913,70 +9447,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX9-DPP-NEXT: .LBB14_3:
@@ -10013,61 +9549,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1064-DPP-NEXT: .LBB14_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -10102,56 +9646,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1032-DPP-NEXT: .LBB14_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -10177,72 +9723,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1164-DPP-NEXT: .LBB14_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -10268,64 +9828,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB14_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB14_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB14_2
; GFX1132-DPP-NEXT: .LBB14_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -10366,50 +9926,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB15_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB15_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB15_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_4
; GFX7LESS-NEXT: .LBB15_5:
; GFX7LESS-NEXT: s_endpgm
@@ -10444,42 +10004,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB15_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB15_4
; GFX9-NEXT: .LBB15_5:
; GFX9-NEXT: s_endpgm
@@ -10514,42 +10076,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB15_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB15_4
; GFX1064-NEXT: .LBB15_5:
; GFX1064-NEXT: s_endpgm
@@ -10584,41 +10146,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB15_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB15_4
; GFX1032-NEXT: .LBB15_5:
; GFX1032-NEXT: s_endpgm
@@ -10643,46 +10205,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB15_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB15_4
; GFX1164-NEXT: .LBB15_5:
; GFX1164-NEXT: s_endpgm
@@ -10708,44 +10270,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB15_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB15_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB15_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB15_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB15_4
; GFX1132-NEXT: .LBB15_5:
; GFX1132-NEXT: s_endpgm
@@ -10836,70 +10398,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX9-DPP-NEXT: .LBB15_3:
@@ -10936,61 +10500,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1064-DPP-NEXT: .LBB15_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11025,56 +10597,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1032-DPP-NEXT: .LBB15_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -11100,72 +10674,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1164-DPP-NEXT: .LBB15_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -11191,64 +10779,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB15_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB15_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB15_2
; GFX1132-DPP-NEXT: .LBB15_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11259,48 +10847,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp(ptr addrspace(1) %ptr) #2 {
; GFX7LESS-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-NEXT: .LBB16_3:
@@ -11308,43 +10889,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX9-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-NEXT: s_mov_b32 s14, -1
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB16_3
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-NEXT: .LBB16_3:
; GFX9-NEXT: s_endpgm
@@ -11352,40 +10926,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1064-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-NEXT: .LBB16_3:
; GFX1064-NEXT: s_endpgm
@@ -11393,171 +10962,153 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-NEXT: s_mov_b32 s14, -1
-; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s6, 0
+; GFX1032-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-NEXT: .LBB16_3:
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164: ; %bb.0:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_clause 0x1
-; GFX1164-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-NEXT: .LBB16_3:
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_clause 0x1
-; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-NEXT: s_mov_b32 s6, 0
+; GFX1132-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-NEXT: .LBB16_3:
; GFX1132-NEXT: s_endpgm
;
; GFX7LESS-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX7LESS-DPP: ; %bb.0:
-; GFX7LESS-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX7LESS-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX7LESS-DPP-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-DPP-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-DPP-NEXT: s_mov_b32 s15, 0xe8f000
-; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
-; GFX7LESS-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-DPP-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7LESS-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX7LESS-DPP-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-DPP-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7LESS-DPP-NEXT: v_readfirstlane_b32 s7, v1
; GFX7LESS-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX7LESS-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX7LESS-DPP-NEXT: ; %bb.1:
; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-DPP-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], s[2:3], v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b64 s[4:5], 0
-; GFX7LESS-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
; GFX7LESS-DPP-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s6
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX7LESS-DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
+; GFX7LESS-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, s8
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, s9
; GFX7LESS-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX7LESS-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[6:7]
; GFX7LESS-DPP-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-DPP-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v6
+; GFX7LESS-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v2, v4
; GFX7LESS-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v7
+; GFX7LESS-DPP-NEXT: v_mov_b32_e32 v3, v5
; GFX7LESS-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7LESS-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX7LESS-DPP-NEXT: .LBB16_3:
@@ -11565,43 +11116,36 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX9-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX9-DPP: ; %bb.0:
-; GFX9-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX9-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX9-DPP-NEXT: s_mov_b32 s14, -1
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-DPP-NEXT: s_mov_b32 s15, 0xe00000
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX9-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX9-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX9-DPP-NEXT: ; %bb.1:
-; GFX9-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v1, 0xc3300000
-; GFX9-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], s[0:1], v[0:1]
-; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX9-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX9-DPP-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-DPP-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-DPP-NEXT: v_mov_b32_e32 v3, s7
; GFX9-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-DPP-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX9-DPP-NEXT: .LBB16_3:
; GFX9-DPP-NEXT: s_endpgm
@@ -11609,40 +11153,35 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1064-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1064-DPP: ; %bb.0:
; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1064-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1064-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1064-DPP-NEXT: s_mov_b32 s15, 0x31e16000
-; GFX1064-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
-; GFX1064-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1064-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
-; GFX1064-DPP-NEXT: s_mov_b32 s3, 0x43300000
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[2:3]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX1064-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1064-DPP-NEXT: .LBB16_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -11650,123 +11189,112 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
-; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
-; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1032-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, s5
; GFX1032-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1032-DPP-NEXT: .LBB16_3:
; GFX1032-DPP-NEXT: s_endpgm
;
; GFX1164-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1164-DPP: ; %bb.0:
-; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, exec
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, s0
-; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_clause 0x1
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1164-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1164-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-DPP-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-DPP-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1164-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s3
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1164-DPP-NEXT: .LBB16_3:
; GFX1164-DPP-NEXT: s_endpgm
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_clause 0x1
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
-; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
-; GFX1132-DPP-NEXT: scratch_load_b64 v[0:1], off, off
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1132-DPP-NEXT: s_mov_b32 s6, 0
+; GFX1132-DPP-NEXT: v_cvt_f64_i32_e32 v[0:1], s0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mul_f64 v[0:1], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_readfirstlane_b32 s1, v1
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
+; GFX1132-DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX1132-DPP-NEXT: .LBB16_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s6, vcc_lo, s6
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB16_2
; GFX1132-DPP-NEXT: .LBB16_3:
; GFX1132-DPP-NEXT: s_endpgm
@@ -11806,50 +11334,50 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX7LESS-NEXT: ; implicit-def: $sgpr15
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v2
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
+; GFX7LESS-NEXT: s_brev_b32 s5, 1
; GFX7LESS-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS-NEXT: v_mov_b32_e32 v4, 0
-; GFX7LESS-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX7LESS-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s4
-; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s4
-; GFX7LESS-NEXT: s_lshl_b64 s[4:5], 1, s4
-; GFX7LESS-NEXT: s_andn2_b64 s[0:1], s[0:1], s[4:5]
-; GFX7LESS-NEXT: v_cmp_ne_u64_e64 s[4:5], s[0:1], 0
-; GFX7LESS-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX7LESS-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GFX7LESS-NEXT: s_cbranch_vccnz .LBB17_1
-; GFX7LESS-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7LESS-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX7LESS-NEXT: v_readlane_b32 s2, v0, s6
+; GFX7LESS-NEXT: v_readlane_b32 s3, v1, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS-NEXT: v_add_f64 v[3:4], s[2:3], v[3:4]
+; GFX7LESS-NEXT: s_bitset0_b64 s[0:1], s6
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v3
+; GFX7LESS-NEXT: s_or_b64 s[2:3], s[0:1], s[0:1]
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v4
+; GFX7LESS-NEXT: s_cbranch_scc1 .LBB17_1
+; GFX7LESS-NEXT: ; %bb.2:
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7LESS-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX7LESS-NEXT: s_cbranch_execz .LBB17_5
; GFX7LESS-NEXT: ; %bb.3:
; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x9
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: s_mov_b64 s[4:5], 0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0
; GFX7LESS-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
+; GFX7LESS-NEXT: v_add_f64 v[0:1], v[2:3], -s[4:5]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v9, v3
-; GFX7LESS-NEXT: v_mov_b32_e32 v8, v2
-; GFX7LESS-NEXT: v_mov_b32_e32 v7, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[0:3], 0 glc
+; GFX7LESS-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v6
-; GFX7LESS-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v7
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v4
+; GFX7LESS-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB17_4
; GFX7LESS-NEXT: .LBB17_5:
; GFX7LESS-NEXT: s_endpgm
@@ -11884,42 +11412,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX9-NEXT: ; implicit-def: $sgpr15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX9-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX9-NEXT: v_readlane_b32 s3, v1, s4
-; GFX9-NEXT: v_readlane_b32 s2, v0, s4
-; GFX9-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX9-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX9-NEXT: s_mov_b32 s0, 0
+; GFX9-NEXT: s_brev_b32 s1, 1
+; GFX9-NEXT: s_mov_b64 s[2:3], exec
+; GFX9-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX9-NEXT: v_readlane_b32 s4, v0, s6
+; GFX9-NEXT: v_readlane_b32 s5, v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_add_f64 v[3:4], s[4:5], v[3:4]
+; GFX9-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX9-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v3
+; GFX9-NEXT: v_readfirstlane_b32 s1, v4
; GFX9-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX9-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9-NEXT: ; %bb.2:
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB17_5
; GFX9-NEXT: ; %bb.3:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX9-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX9-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB17_4
; GFX9-NEXT: .LBB17_5:
; GFX9-NEXT: s_endpgm
@@ -11954,42 +11484,42 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1064-NEXT: ; implicit-def: $sgpr15
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1064-NEXT: v_mov_b32_e32 v4, 0
-; GFX1064-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1064-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064-NEXT: s_ff1_i32_b64 s4, s[0:1]
-; GFX1064-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1064-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1064-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1064-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1064-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1064-NEXT: s_mov_b32 s0, 0
+; GFX1064-NEXT: s_brev_b32 s1, 1
+; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1064-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: s_ff1_i32_b64 s6, s[2:3]
+; GFX1064-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1064-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1064-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1064-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1064-NEXT: v_readfirstlane_b32 s1, v4
; GFX1064-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1064-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064-NEXT: ; %bb.2:
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB17_5
; GFX1064-NEXT: ; %bb.3:
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-NEXT: v_mov_b32_e32 v6, 0
-; GFX1064-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-NEXT: v_mov_b32_e32 v4, 0
+; GFX1064-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1064-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1064-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1064-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1064-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1064-NEXT: v_mov_b32_e32 v3, v1
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
-; GFX1064-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_cbranch_execnz .LBB17_4
; GFX1064-NEXT: .LBB17_5:
; GFX1064-NEXT: s_endpgm
@@ -12024,41 +11554,41 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-NEXT: ; implicit-def: $sgpr15
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-NEXT: v_mov_b32_e32 v4, 0
-; GFX1032-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: s_ff1_i32_b32 s1, s0
-; GFX1032-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1032-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1032-NEXT: s_andn2_b32 s0, s0, s1
-; GFX1032-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s0, 0
+; GFX1032-NEXT: s_brev_b32 s1, 1
+; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1032-NEXT: s_bitset0_b32 s2, s3
+; GFX1032-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1032-NEXT: v_readfirstlane_b32 s1, v4
; GFX1032-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1032-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032-NEXT: ; %bb.2:
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB17_5
; GFX1032-NEXT: ; %bb.3:
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-NEXT: v_mov_b32_e32 v4, 0
+; GFX1032-NEXT: s_mov_b32 s4, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
+; GFX1032-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3]
; GFX1032-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1032-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1032-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1032-NEXT: v_mov_b32_e32 v3, v1
; GFX1032-NEXT: v_mov_b32_e32 v2, v0
-; GFX1032-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-NEXT: s_cbranch_execnz .LBB17_4
; GFX1032-NEXT: .LBB17_5:
; GFX1032-NEXT: s_endpgm
@@ -12083,46 +11613,46 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-NEXT: ; implicit-def: $sgpr15
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1164-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: s_ctz_i32_b64 s4, s[0:1]
-; GFX1164-NEXT: v_readlane_b32 s3, v1, s4
-; GFX1164-NEXT: v_readlane_b32 s2, v0, s4
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1164-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
-; GFX1164-NEXT: s_lshl_b64 s[2:3], 1, s4
-; GFX1164-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164-NEXT: s_mov_b32 s0, 0
+; GFX1164-NEXT: s_brev_b32 s1, 1
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
+; GFX1164-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_ctz_i32_b64 s6, s[2:3]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-NEXT: v_readlane_b32 s4, v0, s6
+; GFX1164-NEXT: v_readlane_b32 s5, v1, s6
+; GFX1164-NEXT: s_bitset0_b64 s[2:3], s6
+; GFX1164-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1164-NEXT: v_readfirstlane_b32 s1, v4
; GFX1164-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1164-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164-NEXT: ; %bb.2:
+; GFX1164-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1164-NEXT: s_cbranch_execz .LBB17_5
; GFX1164-NEXT: ; %bb.3:
-; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-NEXT: v_mov_b32_e32 v6, 0
-; GFX1164-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1164-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1164-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1164-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX1164-NEXT: v_mov_b32_e32 v3, v1
; GFX1164-NEXT: v_mov_b32_e32 v2, v0
-; GFX1164-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_cbranch_execnz .LBB17_4
; GFX1164-NEXT: .LBB17_5:
; GFX1164-NEXT: s_endpgm
@@ -12148,44 +11678,44 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-NEXT: ; implicit-def: $sgpr15
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: .LBB17_1: ; %ComputeLoop
-; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: s_mov_b32 s0, 0
+; GFX1132-NEXT: s_brev_b32 s1, 1
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132-NEXT: s_ctz_i32_b32 s1, s0
-; GFX1132-NEXT: v_readlane_b32 s3, v1, s1
-; GFX1132-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1132-NEXT: s_lshl_b32 s1, 1, s1
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 s0, s0, s1
-; GFX1132-NEXT: v_add_f64 v[4:5], v[4:5], s[2:3]
+; GFX1132-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132-NEXT: v_readlane_b32 s5, v1, s3
+; GFX1132-NEXT: s_bitset0_b32 s2, s3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132-NEXT: v_add_f64 v[3:4], s[4:5], s[0:1]
+; GFX1132-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_readfirstlane_b32 s1, v4
; GFX1132-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX1132-NEXT: ; %bb.2: ; %ComputeEnd
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_mov_b32 s2, 0
-; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132-NEXT: ; %bb.2:
+; GFX1132-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB17_5
; GFX1132-NEXT: ; %bb.3:
-; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-NEXT: v_mov_b32_e32 v4, 0
+; GFX1132-NEXT: s_mov_b32 s4, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
+; GFX1132-NEXT: global_load_b64 v[2:3], v4, s[2:3]
; GFX1132-NEXT: .LBB17_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
+; GFX1132-NEXT: v_add_f64 v[0:1], v[2:3], -s[0:1]
+; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v4, v[0:3], s[2:3] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
; GFX1132-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1132-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-NEXT: s_cbranch_execnz .LBB17_4
; GFX1132-NEXT: .LBB17_5:
; GFX1132-NEXT: s_endpgm
@@ -12276,70 +11806,72 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v6, v4, v1, s[0:1]
-; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
-; GFX9-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_bfrev_b32_e32 v3, 1
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX9-DPP-NEXT: v_cndmask_b32_e64 v5, v3, v1, s[0:1]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: v_mov_b32_e32 v7, v5
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:2 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v7 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[4:5], v[6:7]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:4 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_shr:8 row_mask:0xf bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: v_bfrev_b32_e32 v8, 1
-; GFX9-DPP-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
; GFX9-DPP-NEXT: s_nop 0
-; GFX9-DPP-NEXT: v_mov_b32_dpp v8, v6 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v7, v5 row_bcast:15 row_mask:0xa bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[5:6], v[5:6], v[7:8]
-; GFX9-DPP-NEXT: s_nop 1
-; GFX9-DPP-NEXT: v_mov_b32_dpp v4, v6 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_mov_b32_dpp v3, v5 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-DPP-NEXT: s_nop 0
+; GFX9-DPP-NEXT: v_mov_b32_dpp v5, v5 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_mov_b32_dpp v6, v6 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
-; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX9-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX9-DPP-NEXT: v_readlane_b32 s3, v4, 63
; GFX9-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX9-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
-; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX9-DPP-NEXT: ; %bb.1:
; GFX9-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX9-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX9-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DPP-NEXT: global_load_dwordx2 v[11:12], v0, s[2:3]
+; GFX9-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX9-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_add_f64 v[9:10], v[11:12], -s[0:1]
-; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[9:12], s[2:3] glc
+; GFX9-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX9-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[11:12]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v12, v2
+; GFX9-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v2
; GFX9-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-DPP-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-DPP-NEXT: v_mov_b32_e32 v10, v1
; GFX9-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX9-DPP-NEXT: .LBB17_3:
@@ -12376,61 +11908,69 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s[0:1]
-; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[0:1]
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s[0:1]
+; GFX1064-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1064-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 0
-; GFX1064-DPP-NEXT: v_readlane_b32 s5, v4, 32
-; GFX1064-DPP-NEXT: v_readlane_b32 s4, v3, 32
-; GFX1064-DPP-NEXT: v_add_f64 v[3:4], s[2:3], s[4:5]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v5, -1, v7
+; GFX1064-DPP-NEXT: v_add_nc_u32_e32 v5, 32, v5
+; GFX1064-DPP-NEXT: v_mul_lo_u32 v5, 4, v5
+; GFX1064-DPP-NEXT: ds_permute_b32 v6, v5, v3
+; GFX1064-DPP-NEXT: ds_permute_b32 v7, v5, v4
+; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[6:7]
; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
; GFX1064-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v7, exec_hi, v0
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v7
-; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1064-DPP-NEXT: v_readlane_b32 s2, v3, 63
+; GFX1064-DPP-NEXT: v_readlane_b32 s3, v4, 63
+; GFX1064-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1064-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1064-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064-DPP-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX1064-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1064-DPP-NEXT: ; %bb.1:
-; GFX1064-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
-; GFX1064-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1064-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1064-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1064-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1064-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1064-DPP-NEXT: global_load_dwordx2 v[10:11], v0, s[2:3]
; GFX1064-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1064-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1064-DPP-NEXT: v_add_f64 v[8:9], v[10:11], -s[0:1]
+; GFX1064-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[8:11], s[2:3] glc
; GFX1064-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1064-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1064-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX1064-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[1:2], v[10:11]
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v11, v2
+; GFX1064-DPP-NEXT: v_mov_b32_e32 v10, v1
+; GFX1064-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX1064-DPP-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX1064-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1064-DPP-NEXT: .LBB17_3:
; GFX1064-DPP-NEXT: s_endpgm
@@ -12465,56 +12005,58 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v6, 0x80000000, v1, s0
-; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v5, 0, v0, s0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v4, v6 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[5:6], v[3:4]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v3, 0, v0, s2
+; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v1, s2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:2 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v6, 1
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, 0
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v4 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v6, v4, 0, 0
-; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1032-DPP-NEXT: v_mov_b32_dpp v6, v6 row_shr:8 row_mask:0xf bank_mask:0xf
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
-; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
-; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v7
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: ds_swizzle_b32 v6, v4 offset:swizzle(BROADCAST,32,15)
+; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
+; GFX1032-DPP-NEXT: v_readlane_b32 s0, v3, 31
+; GFX1032-DPP-NEXT: v_readlane_b32 s1, v4, 31
+; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s2, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[2:3], s[34:35], 0x24
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v2, s[0:1]
+; GFX1032-DPP-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX1032-DPP-NEXT: global_load_dwordx2 v[9:10], v0, s[2:3]
; GFX1032-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1032-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -v[0:1]
-; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[7:8], v2, v[7:10], s[0:1] glc
+; GFX1032-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1032-DPP-NEXT: global_atomic_cmpswap_x2 v[1:2], v0, v[7:10], s[2:3] glc
; GFX1032-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[9:10]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v8
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1032-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX1032-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[1:2], v[9:10]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v10, v2
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX1032-DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX1032-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1032-DPP-NEXT: .LBB17_3:
; GFX1032-DPP-NEXT: s_endpgm
@@ -12540,72 +12082,86 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s[0:1]
-; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[0:1]
+; GFX1164-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s[0:1]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1164-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1164-DPP-NEXT: v_permlane64_b32 v5, v3
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_permlane64_b32 v4, v2
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v4, -1, v4
+; GFX1164-DPP-NEXT: v_add_nc_u32_e32 v4, 32, v4
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_mul_lo_u32 v4, 4, v4
+; GFX1164-DPP-NEXT: ds_permute_b32 v5, v4, v2
+; GFX1164-DPP-NEXT: ds_permute_b32 v6, v4, v3
+; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[5:6]
; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
-; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, 0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v1, v3
-; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v6, exec_hi, v0
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1164-DPP-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX1164-DPP-NEXT: v_readlane_b32 s2, v2, 63
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1164-DPP-NEXT: v_readlane_b32 s3, v3, 63
; GFX1164-DPP-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
+; GFX1164-DPP-NEXT: s_mov_b64 exec, s[0:1]
+; GFX1164-DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164-DPP-NEXT: s_mov_b64 s[0:1], s[2:3]
+; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1164-DPP-NEXT: ; %bb.1:
-; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
-; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX1164-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v0, 0
+; GFX1164-DPP-NEXT: s_mov_b64 s[4:5], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1164-DPP-NEXT: global_load_b64 v[9:10], v0, s[2:3]
; GFX1164-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1164-DPP-NEXT: v_add_f64 v[7:8], v[9:10], -s[0:1]
+; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[7:8], v0, v[7:10], s[2:3] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
+; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[9:10]
+; GFX1164-DPP-NEXT: v_mov_b32_e32 v10, v8
; GFX1164-DPP-NEXT: v_mov_b32_e32 v9, v7
-; GFX1164-DPP-NEXT: v_mov_b32_e32 v8, v6
-; GFX1164-DPP-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1164-DPP-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX1164-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1164-DPP-NEXT: s_and_not1_b64 exec, exec, s[4:5]
; GFX1164-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1164-DPP-NEXT: .LBB17_3:
; GFX1164-DPP-NEXT: s_endpgm
@@ -12631,64 +12187,64 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
-; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x80000000, v1, s0
-; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v4, 0, v0, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v3, v5 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v2, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[4:5], v[2:3]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:2 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_or_saveexec_b32 s2, -1
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
+; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v3, 0x80000000, v1, s2
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:4 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v5, 1
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, 0
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v3 row_xmask:8 row_mask:0xf bank_mask:0xf
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v2 row_xmask:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v5, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
-; GFX1132-DPP-NEXT: v_permlanex16_b32 v4, v2, 0, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v4, v4 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: v_mov_b32_dpp v5, v5 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
-; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v6
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v4, v2 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: ds_swizzle_b32 v5, v3 offset:swizzle(BROADCAST,32,15)
+; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
+; GFX1132-DPP-NEXT: v_readlane_b32 s0, v2, 31
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_readlane_b32 s1, v3, 31
+; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s2
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s4, 0
+; GFX1132-DPP-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB17_3
; GFX1132-DPP-NEXT: ; %bb.1:
-; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: s_load_b64 s[2:3], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
+; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v0, s[2:3]
; GFX1132-DPP-NEXT: .LBB17_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -v[0:1]
-; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
+; GFX1132-DPP-NEXT: v_add_f64 v[6:7], v[8:9], -s[0:1]
+; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v0, v[6:9], s[2:3] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
; GFX1132-DPP-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
-; GFX1132-DPP-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX1132-DPP-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX1132-DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX1132-DPP-NEXT: s_cbranch_execnz .LBB17_2
; GFX1132-DPP-NEXT: .LBB17_3:
; GFX1132-DPP-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index ce761e1d59cb3..dae9c8177bc9b 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -683,19 +683,19 @@ main_body:
define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_local:
; GFX9: ; %bb.0:
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-NEXT: s_mul_i32 s0, s0, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
; GFX9-NEXT: ds_add_u32 v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB5_2:
@@ -703,19 +703,19 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX90A-LABEL: atomic_add_local:
; GFX90A: ; %bb.0:
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB5_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX90A-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX90A-NEXT: s_mul_i32 s0, s0, 5
; GFX90A-NEXT: v_mov_b32_e32 v1, s0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NEXT: v_mov_b32_e32 v0, s1
; GFX90A-NEXT: ds_add_u32 v0, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB5_2:
@@ -723,15 +723,15 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX10-LABEL: atomic_add_local:
; GFX10: ; %bb.0:
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX10-NEXT: s_mov_b32 s0, exec_lo
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: s_mul_i32 s0, s0, 5
; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: v_mov_b32_e32 v0, s1
@@ -743,19 +743,19 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX9-FLATSCR-LABEL: atomic_add_local:
; GFX9-FLATSCR: ; %bb.0:
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB5_2
; GFX9-FLATSCR-NEXT: ; %bb.1:
-; GFX9-FLATSCR-NEXT: s_load_dword s2, s[4:5], 0x24
+; GFX9-FLATSCR-NEXT: s_load_dword s1, s[4:5], 0x24
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
; GFX9-FLATSCR-NEXT: s_mul_i32 s0, s0, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s1
; GFX9-FLATSCR-NEXT: ds_add_u32 v0, v1
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: .LBB5_2:
@@ -763,18 +763,18 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX11-LABEL: atomic_add_local:
; GFX11: ; %bb.0:
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB5_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b32 s1, s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s0, s0, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX11-NEXT: ds_add_u32 v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -784,19 +784,18 @@ define amdgpu_kernel void @atomic_add_local(ptr addrspace(3) %local) {
;
; GFX12-LABEL: atomic_add_local:
; GFX12: ; %bb.0:
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB5_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b32 s1, s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s0, s0, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v0, s1
; GFX12-NEXT: ds_add_u32 v0, v1
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -883,20 +882,20 @@ define void @flat_atomic_xchg_i32_noret(ptr %ptr, i32 %in) {
define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_ret_local:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB7_2:
@@ -912,20 +911,20 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX90A-LABEL: atomic_add_ret_local:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_mov_b64 s[2:3], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB7_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX90A-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: s_mul_i32 s2, s2, 5
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s3
; GFX90A-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB7_2:
@@ -941,16 +940,16 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX10-LABEL: atomic_add_ret_local:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB7_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT: s_mul_i32 s1, s1, 5
; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: v_mov_b32_e32 v1, s2
@@ -971,20 +970,20 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX9-FLATSCR-LABEL: atomic_add_ret_local:
; GFX9-FLATSCR: ; %bb.0:
-; GFX9-FLATSCR-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB7_2
; GFX9-FLATSCR-NEXT: ; %bb.1:
-; GFX9-FLATSCR-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-FLATSCR-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT: s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s3
; GFX9-FLATSCR-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: .LBB7_2:
@@ -1000,19 +999,19 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX11-LABEL: atomic_add_ret_local:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB7_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1031,20 +1030,19 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX12-LABEL: atomic_add_ret_local:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB7_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s1, s1, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
; GFX12-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -1075,9 +1073,10 @@ declare i32 @llvm.amdgcn.raw.ptr.buffer.atomic.add(i32, ptr addrspace(8), i32, i
define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace(8) %inout) {
; GFX9-LABEL: add_i32_constant:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1085,7 +1084,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_mul_i32 s2, s2, 5
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1103,9 +1101,10 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX90A-LABEL: add_i32_constant:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: s_mov_b64 s[2:3], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1113,7 +1112,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT: s_mul_i32 s2, s2, 5
; GFX90A-NEXT: v_mov_b32_e32 v1, s2
; GFX90A-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1131,16 +1129,16 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX10-LABEL: add_i32_constant:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s1, s0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB8_2
; GFX10-NEXT: ; %bb.1:
; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX10-NEXT: s_mul_i32 s1, s1, 5
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1159,9 +1157,10 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX9-FLATSCR-LABEL: add_i32_constant:
; GFX9-FLATSCR: ; %bb.0: ; %entry
-; GFX9-FLATSCR-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX9-FLATSCR-NEXT: s_mov_b64 s[0:1], exec
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[0:1]
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: ; implicit-def: $vgpr1
; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[0:1], vcc
@@ -1169,7 +1168,6 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-FLATSCR-NEXT: ; %bb.1:
; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-FLATSCR-NEXT: s_mul_i32 s2, s2, 5
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s2
; GFX9-FLATSCR-NEXT: buffer_atomic_add v1, off, s[8:11], 0 glc
@@ -1187,19 +1185,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX11-LABEL: add_i32_constant:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB8_2
; GFX11-NEXT: ; %bb.1:
; GFX11-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_i32 s1, s1, 5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], 0 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -1217,20 +1215,19 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
;
; GFX12-LABEL: add_i32_constant:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s1, s0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB8_2
; GFX12-NEXT: ; %bb.1:
; GFX12-NEXT: s_load_b128 s[8:11], s[4:5], 0x34
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mul_i32 s1, s1, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_mov_b32_e32 v1, s1
; GFX12-NEXT: buffer_atomic_add_u32 v1, off, s[8:11], null th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 99a6a5cd346e2..2856368268886 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -7712,44 +7712,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX12-LABEL: local_ds_fadd:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_add_co_i32 s1, s3, 4
-; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX12-NEXT: s_cbranch_execz .LBB28_2
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX12-NEXT: s_lshl_b32 s6, s1, 3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_2:
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: s_mov_b32 s7, exec_lo
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX12-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX12-NEXT: s_cbranch_execz .LBB28_4
; GFX12-NEXT: ; %bb.3:
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX12-NEXT: s_lshl_b32 s0, s1, 4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: ds_add_f32 v2, v1
+; GFX12-NEXT: ds_add_f32 v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_4:
@@ -7807,47 +7814,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX942-LABEL: local_ds_fadd:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_cbranch_execz .LBB28_2
; GFX942-NEXT: ; %bb.1:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s8
-; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX942-NEXT: s_lshl_b32 s7, s3, 3
+; GFX942-NEXT: v_mov_b32_e32 v1, s7
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
+; GFX942-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_readfirstlane_b32 s10, v1
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_mov_b64 s[0:1], exec
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX942-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
+; GFX942-NEXT: v_readfirstlane_b32 s9, v2
; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB28_4
; GFX942-NEXT: ; %bb.3:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: s_lshl_b32 s0, s3, 4
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: ds_add_f32 v2, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_mov_b32_e32 v2, s9
+; GFX942-NEXT: ds_add_f32 v1, v2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_4:
; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX942-NEXT: v_mov_b32_e32 v1, s10
+; GFX942-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX942-NEXT: v_bfrev_b32_e32 v1, 1
@@ -7891,42 +7902,47 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX11-LABEL: local_ds_fadd:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_i32 s1, s3, 4
-; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX11-NEXT: s_cbranch_execz .LBB28_2
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_lshl_b32 s6, s1, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_2:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_mov_b32 s7, exec_lo
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: v_readfirstlane_b32 s3, v1
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX11-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX11-NEXT: s_cbranch_execz .LBB28_4
; GFX11-NEXT: ; %bb.3:
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX11-NEXT: s_lshl_b32 s0, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_f32 v2, v1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: ds_add_f32 v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_4:
@@ -7979,40 +7995,44 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-LABEL: local_ds_fadd:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_i32 s1, s3, 4
-; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB28_2
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX10-NEXT: s_lshl_b32 s6, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB28_2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_mov_b32 s7, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v2
; GFX10-NEXT: s_and_saveexec_b32 s6, s0
; GFX10-NEXT: s_cbranch_execz .LBB28_4
; GFX10-NEXT: ; %bb.3:
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX10-NEXT: s_lshl_b32 s0, s1, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_add_f32 v2, v1
+; GFX10-NEXT: ds_add_f32 v1, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB28_4:
@@ -8062,47 +8082,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX90A-LABEL: local_ds_fadd:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB28_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX90A-NEXT: s_lshl_b32 s7, s3, 3
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
+; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_mov_b64 s[8:9], exec
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX90A-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v2
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX90A-NEXT: s_cbranch_execz .LBB28_4
; GFX90A-NEXT: ; %bb.3:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-NEXT: ds_add_f32 v2, v1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s9
+; GFX90A-NEXT: ds_add_f32 v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_4:
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s10
+; GFX90A-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8144,47 +8168,51 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX908-LABEL: local_ds_fadd:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX908-NEXT: s_cbranch_execz .LBB28_2
; GFX908-NEXT: ; %bb.1:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX908-NEXT: s_lshl_b32 s7, s3, 3
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_mov_b64 s[8:9], exec
-; GFX908-NEXT: v_readfirstlane_b32 s10, v1
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX908-NEXT: s_mov_b64 s[0:1], exec
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX908-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v2
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX908-NEXT: s_cbranch_execz .LBB28_4
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: s_lshl_b32 s0, s3, 4
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s0
-; GFX908-NEXT: ds_add_f32 v2, v1
+; GFX908-NEXT: v_mov_b32_e32 v1, s0
+; GFX908-NEXT: v_mov_b32_e32 v2, s9
+; GFX908-NEXT: ds_add_f32 v1, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_4:
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX908-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8226,48 +8254,52 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX8-LABEL: local_ds_fadd:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB28_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
+; GFX8-NEXT: s_lshl_b32 s7, s3, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
-; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v2
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX8-NEXT: s_cbranch_execz .LBB28_4
; GFX8-NEXT: ; %bb.3:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, s3, 4
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: ds_add_f32 v2, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
+; GFX8-NEXT: ds_add_f32 v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_4:
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
+; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8311,33 +8343,34 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX7-LABEL: local_ds_fadd:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX7-NEXT: s_mov_b64 s[0:1], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB28_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: s_lshl_b32 s0, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: ds_read_b32 v1, v2
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_2
@@ -8345,30 +8378,31 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: .LBB28_4: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_mov_b64 s[8:9], exec
+; GFX7-NEXT: s_mov_b64 s[0:1], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v2
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s0, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: ds_read_b32 v3, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX7-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
; GFX7-NEXT: .LBB28_7: ; %Flow21
@@ -8435,33 +8469,34 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
;
; GFX6-LABEL: local_ds_fadd:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX6-NEXT: s_mov_b64 s[0:1], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_add_i32 s3, s3, 4
+; GFX6-NEXT: v_readfirstlane_b32 s10, v1
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB28_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s8, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: s_lshl_b32 s0, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_read_b32 v1, v2
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, v1
+; GFX6-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_2
@@ -8469,30 +8504,31 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: .LBB28_4: ; %Flow21
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_mov_b64 s[8:9], exec
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX6-NEXT: v_readfirstlane_b32 s10, v1
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v2
; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB28_7
; GFX6-NEXT: ; %bb.5:
; GFX6-NEXT: s_lshl_b32 s0, s3, 4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
-; GFX6-NEXT: ds_read_b32 v3, v1
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: ds_read_b32 v2, v1
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX6-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_6
; GFX6-NEXT: .LBB28_7: ; %Flow19
@@ -8572,45 +8608,49 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX12-LABEL: local_ds_fadd_one_as:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_add_co_i32 s1, s3, 4
-; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX12-NEXT: s_cbranch_execz .LBB29_2
+; GFX12-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX12-NEXT: .LBB29_2:
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_lshl_b32 s6, s1, 3
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX12-NEXT: ; %bb.2:
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_mov_b32 s7, exec_lo
+; GFX12-NEXT: s_mov_b32 s0, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX12-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX12-NEXT: s_cbranch_execz .LBB29_4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX12-NEXT: v_readfirstlane_b32 s7, v2
+; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX12-NEXT: ; %bb.3:
-; GFX12-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX12-NEXT: s_lshl_b32 s0, s1, 4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX12-NEXT: ds_add_f32 v2, v1
-; GFX12-NEXT: .LBB29_4:
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: ds_add_f32 v1, v2
+; GFX12-NEXT: ; %bb.4:
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX12-NEXT: s_mov_b32 s1, exec_lo
@@ -8661,46 +8701,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX942-LABEL: local_ds_fadd_one_as:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB29_2
+; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: ; %bb.1:
+; GFX942-NEXT: s_lshl_b32 s7, s3, 3
+; GFX942-NEXT: v_mov_b32_e32 v1, s7
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
+; GFX942-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX942-NEXT: ; %bb.2:
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s8
-; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX942-NEXT: .LBB29_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX942-NEXT: s_mov_b64 s[8:9], exec
+; GFX942-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s10, v1
-; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX942-NEXT: v_readfirstlane_b32 s8, v1
+; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX942-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
+; GFX942-NEXT: v_readfirstlane_b32 s9, v2
; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX942-NEXT: s_cbranch_execz .LBB29_4
; GFX942-NEXT: ; %bb.3:
-; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: s_lshl_b32 s0, s3, 4
-; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: ds_add_f32 v2, v1
-; GFX942-NEXT: .LBB29_4:
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_mov_b32_e32 v2, s9
+; GFX942-NEXT: ds_add_f32 v1, v2
+; GFX942-NEXT: ; %bb.4:
; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX942-NEXT: v_mov_b32_e32 v1, s10
+; GFX942-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, s8
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX942-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8742,42 +8784,45 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX11-LABEL: local_ds_fadd_one_as:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX11-NEXT: v_readfirstlane_b32 s0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_i32 s1, s3, 4
-; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-NEXT: s_cbranch_execz .LBB29_2
+; GFX11-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_lshl_b32 s6, s1, 3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX11-NEXT: .LBB29_2:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_mov_b32 s7, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX11-NEXT: ; %bb.2:
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX11-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s3, v1
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX11-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX11-NEXT: s_mov_b32 s6, exec_lo
-; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX11-NEXT: s_cbranch_execz .LBB29_4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX11-NEXT: v_readfirstlane_b32 s7, v2
+; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v3
; GFX11-NEXT: ; %bb.3:
-; GFX11-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX11-NEXT: s_lshl_b32 s0, s1, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
-; GFX11-NEXT: ds_add_f32 v2, v1
-; GFX11-NEXT: .LBB29_4:
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s0
+; GFX11-NEXT: ds_add_f32 v1, v2
+; GFX11-NEXT: ; %bb.4:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8824,39 +8869,41 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX10-LABEL: local_ds_fadd_one_as:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
-; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT: ; implicit-def: $vgpr1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_add_i32 s1, s3, 4
-; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX10-NEXT: s_cbranch_execz .LBB29_2
+; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX10-NEXT: .LBB29_2:
+; GFX10-NEXT: s_lshl_b32 s6, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_mov_b32 s7, exec_lo
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT: s_mov_b32 s0, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v3, exec_lo, 0
+; GFX10-NEXT: s_bcnt1_i32_b32 s0, s0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
+; GFX10-NEXT: v_cvt_f32_i32_e32 v2, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v3
+; GFX10-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX10-NEXT: v_readfirstlane_b32 s7, v2
; GFX10-NEXT: s_and_saveexec_b32 s6, s0
-; GFX10-NEXT: s_cbranch_execz .LBB29_4
; GFX10-NEXT: ; %bb.3:
-; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX10-NEXT: s_lshl_b32 s0, s1, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX10-NEXT: ds_add_f32 v2, v1
-; GFX10-NEXT: .LBB29_4:
+; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: ds_add_f32 v1, v2
+; GFX10-NEXT: ; %bb.4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
@@ -8899,46 +8946,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX90A-LABEL: local_ds_fadd_one_as:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX90A-NEXT: s_cbranch_execz .LBB29_2
+; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: ; %bb.1:
+; GFX90A-NEXT: s_lshl_b32 s7, s3, 3
+; GFX90A-NEXT: v_mov_b32_e32 v1, s7
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
+; GFX90A-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX90A-NEXT: ; %bb.2:
+; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s8
-; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX90A-NEXT: .LBB29_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: s_mov_b64 s[8:9], exec
+; GFX90A-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX90A-NEXT: v_readfirstlane_b32 s8, v1
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX90A-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX90A-NEXT: v_readfirstlane_b32 s9, v2
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX90A-NEXT: s_cbranch_execz .LBB29_4
; GFX90A-NEXT: ; %bb.3:
-; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
-; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s0
-; GFX90A-NEXT: ds_add_f32 v2, v1
-; GFX90A-NEXT: .LBB29_4:
+; GFX90A-NEXT: v_mov_b32_e32 v1, s0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s9
+; GFX90A-NEXT: ds_add_f32 v1, v2
+; GFX90A-NEXT: ; %bb.4:
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s10
+; GFX90A-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s8
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX90A-NEXT: v_bfrev_b32_e32 v1, 1
@@ -8978,46 +9027,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX908-LABEL: local_ds_fadd_one_as:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX908-NEXT: s_cbranch_execz .LBB29_2
+; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX908-NEXT: ; %bb.1:
+; GFX908-NEXT: s_lshl_b32 s7, s3, 3
+; GFX908-NEXT: v_mov_b32_e32 v1, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX908-NEXT: ; %bb.2:
+; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s8
-; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX908-NEXT: .LBB29_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_mov_b64 s[8:9], exec
+; GFX908-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s10, v1
-; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX908-NEXT: v_readfirstlane_b32 s8, v1
+; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX908-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX908-NEXT: v_readfirstlane_b32 s9, v2
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX908-NEXT: s_cbranch_execz .LBB29_4
; GFX908-NEXT: ; %bb.3:
-; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: s_lshl_b32 s0, s3, 4
-; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s0
-; GFX908-NEXT: ds_add_f32 v2, v1
-; GFX908-NEXT: .LBB29_4:
+; GFX908-NEXT: v_mov_b32_e32 v1, s0
+; GFX908-NEXT: v_mov_b32_e32 v2, s9
+; GFX908-NEXT: ds_add_f32 v1, v2
+; GFX908-NEXT: ; %bb.4:
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX908-NEXT: v_mov_b32_e32 v1, s10
+; GFX908-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, s8
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX908-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9057,47 +9108,49 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX8-LABEL: local_ds_fadd_one_as:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
+; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB29_2
+; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_lshl_b32 s7, s3, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: ds_add_rtn_f32 v1, v1, v2
+; GFX8-NEXT: ; %bb.2:
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s8
-; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
-; GFX8-NEXT: .LBB29_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
+; GFX8-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v1
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s9, v2
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
-; GFX8-NEXT: s_cbranch_execz .LBB29_4
; GFX8-NEXT: ; %bb.3:
-; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, s3, 4
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: ds_add_f32 v2, v1
-; GFX8-NEXT: .LBB29_4:
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
+; GFX8-NEXT: ds_add_f32 v1, v2
+; GFX8-NEXT: ; %bb.4:
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
+; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; GFX8-NEXT: v_bfrev_b32_e32 v1, 1
@@ -9139,33 +9192,34 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX7-LABEL: local_ds_fadd_one_as:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX7-NEXT: s_mov_b64 s[0:1], exec
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB29_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: s_lshl_b32 s0, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: ds_read_b32 v1, v2
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_2
@@ -9173,30 +9227,31 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: .LBB29_4: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_mov_b64 s[8:9], exec
+; GFX7-NEXT: s_mov_b64 s[0:1], exec
+; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX7-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_readfirstlane_b32 s11, v2
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_7
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s0, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
-; GFX7-NEXT: ds_read_b32 v3, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX7-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_6
; GFX7-NEXT: .LBB29_7: ; %Flow21
@@ -9263,33 +9318,34 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
;
; GFX6-LABEL: local_ds_fadd_one_as:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
; GFX6-NEXT: s_mov_b64 s[0:1], exec
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_add_i32 s3, s3, 4
+; GFX6-NEXT: v_readfirstlane_b32 s10, v1
; GFX6-NEXT: ; implicit-def: $vgpr1
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB29_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s8, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: s_lshl_b32 s0, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s0
; GFX6-NEXT: ds_read_b32 v1, v2
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
-; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, v1
+; GFX6-NEXT: v_add_f32_e32 v1, s10, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_2
@@ -9297,30 +9353,31 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX6-NEXT: .LBB29_4: ; %Flow21
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX6-NEXT: s_mov_b64 s[8:9], exec
+; GFX6-NEXT: s_mov_b64 s[0:1], exec
+; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0
; GFX6-NEXT: v_readfirstlane_b32 s10, v1
-; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
-; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
+; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, exec_lo, 0
+; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, exec_hi, v1
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_readfirstlane_b32 s11, v2
; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB29_7
; GFX6-NEXT: ; %bb.5:
; GFX6-NEXT: s_lshl_b32 s0, s3, 4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
-; GFX6-NEXT: ds_read_b32 v3, v1
-; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: ds_read_b32 v2, v1
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
+; GFX6-NEXT: v_add_f32_e32 v3, s11, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v3
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_6
; GFX6-NEXT: .LBB29_7: ; %Flow19
More information about the llvm-commits
mailing list